From 7930260feda84ff59acb6459b504ccba88e08c8a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 11 Jun 2022 16:44:01 +0800 Subject: [PATCH 01/90] other: add debug logs when load block info --- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 1c2514d46f..7e3dc5b98f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -233,6 +233,17 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { tsdbError("vgId:%d, SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len); return -1; + } else { + SBlockInfo *pBlkInfo = pReadh->pBlkInfo; + SBlock *pBlk = NULL; + int nBlks = (pBlkIdx->len - sizeof(SBlockInfo)) / sizeof(SBlock); + for (int n = 0; n < nBlks; ++n) { + pBlk = &pBlkInfo->blocks[n]; + if (pBlk->numOfSubBlocks == 1) { + tsdbDebug("prop:vgId:%d, file %s , offset:%u len :%u has %d rows", TSDB_READ_REPO_ID(pReadh), + TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, pBlk->numOfRows); + } + } } // ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid); From 9b8a7d11df194c003006ea000ab5c4bac781cff7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 11 Jun 2022 18:50:28 +0800 Subject: [PATCH 02/90] other: add logs when read blk info --- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 7e3dc5b98f..e4c5ef8fc3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -239,7 +239,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { int nBlks = (pBlkIdx->len - sizeof(SBlockInfo)) / sizeof(SBlock); for (int n = 0; n < nBlks; ++n) { pBlk = &pBlkInfo->blocks[n]; - if (pBlk->numOfSubBlocks == 1) { + if (pBlk->numOfSubBlocks >= 1) { tsdbDebug("prop:vgId:%d, file %s , offset:%u len :%u has %d rows", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, pBlk->numOfRows); } From e3e7c3979eb3f071076cc77e5bd1eec3e02467f2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 20:40:19 +0800 Subject: [PATCH 03/90] show queries --- source/dnode/mnode/impl/src/mndProfile.c | 113 ++++++++--------------- 1 file changed, 36 insertions(+), 77 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 4cfcf389aa..df91ecef25 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -76,7 +76,7 @@ int32_t mndInitProfile(SMnode *pMnode) { // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); - // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); return 0; @@ -619,16 +619,13 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, char *data, int3 return numOfRows; } -static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->info.node; - int32_t numOfRows = 0; -#if 0 - SConnObj *pConn = NULL; - int32_t cols = 0; - char *pWrite; - void *pIter; - char str[TSDB_IPv4ADDR_LEN + 6] = {0}; - +static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SConnObj *pConn = NULL; + if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; pShow->pIter = taosCacheCreateIter(pMgmt->cache); @@ -636,90 +633,52 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, char *data, in while (numOfRows < rows) { pConn = mndGetNextConn(pMnode, pShow->pIter); - if (pConn == NULL) { - pShow->pIter = NULL; - break; + if (pConn == NULL) break; + + if (NULL == pConn->pQueries || taosArrayGetSize(pConn->pQueries) <= 0) { + continue; } - if (numOfRows + pConn->numOfQueries >= rows) { - taosCacheDestroyIter(pShow->pIter); - pShow->pIter = NULL; - break; - } - - for (int32_t i = 0; i < pConn->numOfQueries; ++i) { - SQueryDesc *pDesc = pConn->pQueries + i; + int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); + for (int32_t i = 0; i < numOfQueries; ++i) { cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->queryId); - cols++; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int64_t *)pWrite = htobe64(pConn->id); - cols++; + char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->pMeta->pSchemas[cols].bytes); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, buf, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConn->ip), pConn->port); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->pMeta->pSchemas[cols].bytes); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + int16_t id = mndGetVnodesNum(pMnode, pDnode->id); + colDataAppend(pColInfo, numOfRows, (const char *)&id, false); - char handleBuf[24] = {0}; - snprintf(handleBuf, tListLen(handleBuf), "%" PRIu64, htobe64(pDesc->qId)); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, handleBuf, pShow->pMeta->pSchemas[cols].bytes); - cols++; + char b1[9] = {0}; + STR_TO_VARSTR(b1, online ? "ready" : "offline"); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, b1, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->stime); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->useconds); - cols++; + char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]); - snprintf(str, tListLen(str), "0x%" PRIx64, htobe64(pDesc->sqlObjId)); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->pMeta->pSchemas[cols].bytes); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int32_t *)pWrite = htonl(pDesc->pid); - cols++; - - char epBuf[TSDB_EP_LEN + 1] = {0}; - snprintf(epBuf, tListLen(epBuf), "%s:%u", pDesc->fqdn, pConn->port); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->pMeta->pSchemas[cols].bytes); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(bool *)pWrite = pDesc->stableQuery; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int32_t *)pWrite = htonl(pDesc->numOfSub); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->subSqlInfo, pShow->pMeta->pSchemas[cols].bytes); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->pMeta->pSchemas[cols].bytes); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, b, false); numOfRows++; + sdbRelease(pSdb, pDnode); } } pShow->numOfRows += numOfRows; -#endif return numOfRows; } From f15d17c5789195cfaae3e157cb37e5dc8b010547 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 21:03:40 +0800 Subject: [PATCH 04/90] show queries --- source/common/src/systable.c | 4 ++-- source/dnode/mnode/impl/src/mndProfile.c | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 08977abd61..0c35f5691e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -298,8 +298,8 @@ static const SSysDbTableSchema offsetSchema[] = { }; static const SSysDbTableSchema querySchema[] = { - {.name = "query_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "query_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index df91ecef25..2836473793 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -641,23 +641,25 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); for (int32_t i = 0; i < numOfQueries; ++i) { + SQueryDesc* pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false); - - char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); + colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->queryId, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, buf, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false); + char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(b1, pConn->user); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - int16_t id = mndGetVnodesNum(pMnode, pDnode->id); - colDataAppend(pColInfo, numOfRows, (const char *)&id, false); + colDataAppend(pColInfo, numOfRows, (const char *)user, false); + char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); + varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false); + colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false); char b1[9] = {0}; STR_TO_VARSTR(b1, online ? "ready" : "offline"); From 947aecb74fb2f887ee3ae966176591e7d967905e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 11:27:53 +0800 Subject: [PATCH 05/90] show queries/connection; kill connection --- include/common/systable.h | 1 + include/common/tmsg.h | 5 +- include/libs/nodes/querynodes.h | 1 + include/util/tdef.h | 1 + source/client/inc/clientInt.h | 7 +- source/client/inc/clientStmt.h | 2 +- source/client/src/clientHb.c | 28 ++-- source/client/src/clientImpl.c | 17 ++- source/client/src/clientMain.c | 83 +++++++++--- source/client/src/clientSml.c | 25 ++-- source/client/src/clientStmt.c | 2 +- source/common/src/systable.c | 19 ++- source/common/src/tmsg.c | 6 +- source/dnode/mnode/impl/src/mndProfile.c | 157 ++++++++++++++--------- source/libs/function/src/udfd.c | 4 +- source/libs/nodes/src/nodesUtilFuncs.c | 1 + source/libs/parser/src/parAstParser.c | 14 ++ source/libs/parser/src/parTranslater.c | 17 ++- source/libs/scheduler/src/scheduler.c | 4 +- source/util/src/tcache.c | 8 +- 20 files changed, 271 insertions(+), 131 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 8b0bb4a3fb..d2c28941c7 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -52,6 +52,7 @@ extern "C" { #define TSDB_PERFS_TABLE_OFFSETS "offsets" #define TSDB_PERFS_TABLE_TRANS "trans" #define TSDB_PERFS_TABLE_STREAMS "streams" +#define TSDB_PERFS_TABLE_APPS "apps" typedef struct SSysDbTableSchema { const char* name; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 80e8d29197..81123ea3d9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1982,16 +1982,17 @@ typedef struct { typedef struct { int64_t tid; - int32_t status; + char status[TSDB_JOB_STATUS_LEN]; } SQuerySubDesc; typedef struct { char sql[TSDB_SHOW_SQL_LEN]; uint64_t queryId; int64_t useconds; - int64_t stime; + int64_t stime; // timestamp precision ms int64_t reqRid; int32_t pid; + bool stableQuery; char fqdn[TSDB_FQDN_LEN]; int32_t subPlanNum; SArray* subDesc; // SArray diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 33cc4a9c86..42c7c27951 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -352,6 +352,7 @@ typedef struct SQuery { SArray* pPlaceholderValues; SNode* pPrepareRoot; struct SParseMetaCache* pMetaCache; + bool stableQuery; } SQuery; void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); diff --git a/include/util/tdef.h b/include/util/tdef.h index 1fa8ad1912..73b40c71d8 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -241,6 +241,7 @@ typedef enum ELogicConditionType { #define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 +#define TSDB_JOB_STATUS_LEN 32 #define TSDB_CLUSTER_ID_LEN 40 #define TSDB_FQDN_LEN 128 diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d72aa39e4c..69a24e20cd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 0 +#define SYNC_ON_TOP_OF_ASYNC 1 enum { RES_TYPE__QUERY = 1, @@ -213,6 +213,7 @@ typedef struct SRequestObj { SArray* tableList; SQueryExecMetric metric; SRequestSendRecvBody body; + bool stableQuery; uint32_t prevCode; //previous error code: todo refactor, add update flag for catalog uint32_t retry; @@ -294,7 +295,7 @@ void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); -TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, +STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType); SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); @@ -305,6 +306,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); +void taos_close_internal(void *taos); + // --- heartbeat // global, called by mgmt int hbMgrInit(); diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 936fb92fc4..c2b5d1de6f 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -107,7 +107,7 @@ typedef struct STscStmt { #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) -TAOS_STMT *stmtInit(TAOS *taos); +TAOS_STMT *stmtInit(STscObj* taos); int stmtClose(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt); const char *stmtErrstr(TAOS_STMT *stmt); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e111883381..3ea40627a1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -174,7 +174,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { } if (pRsp->query->killConnection) { - taos_close(pTscObj); + taos_close_internal(pTscObj); } if (pRsp->query->pQnodeList) { @@ -310,11 +310,12 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { } tstrncpy(desc.sql, pRequest->sqlstr, sizeof(desc.sql)); - desc.stime = pRequest->metric.start; + desc.stime = pRequest->metric.start / 1000; desc.queryId = pRequest->requestId; desc.useconds = now - pRequest->metric.start; desc.reqRid = pRequest->self; desc.pid = hbBasic->pid; + desc.stableQuery = pRequest->stableQuery; taosGetFqdn(desc.fqdn); desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0; @@ -329,6 +330,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { if (code) { taosArrayDestroy(desc.subDesc); desc.subDesc = NULL; + desc.subPlanNum = 0; } } else { desc.subDesc = NULL; @@ -350,19 +352,24 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_QRY_APP_ERROR; } - int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; - if (numOfQueries <= 0) { - releaseTscObj(connKey->tscRid); - tscDebug("no queries on connection"); - return TSDB_CODE_QRY_APP_ERROR; - } - SQueryHbReqBasic *hbBasic = (SQueryHbReqBasic *)taosMemoryCalloc(1, sizeof(SQueryHbReqBasic)); if (NULL == hbBasic) { tscError("calloc %d failed", (int32_t)sizeof(SQueryHbReqBasic)); releaseTscObj(connKey->tscRid); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + + hbBasic->connId = pTscObj->connId; + hbBasic->pid = taosGetPId(); + taosGetAppName(hbBasic->app, NULL); + + int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; + if (numOfQueries <= 0) { + req->query = hbBasic; + releaseTscObj(connKey->tscRid); + tscDebug("no queries on connection"); + return TSDB_CODE_SUCCESS; + } hbBasic->queryDesc = taosArrayInit(numOfQueries, sizeof(SQueryDesc)); if (NULL == hbBasic->queryDesc) { @@ -372,9 +379,6 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - hbBasic->connId = pTscObj->connId; - hbBasic->pid = taosGetPId(); - taosGetAppName(hbBasic->app, NULL); int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); if (code) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 216aa5fbd0..949aa22cd1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -58,7 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo, int connType); -TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, +STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, uint16_t port, int connType) { if (taos_init() != TSDB_CODE_SUCCESS) { return NULL; @@ -697,6 +697,8 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return pRequest; } + pRequest->stableQuery = pQuery->stableQuery; + return launchQueryImpl(pRequest, pQuery, false, NULL); } @@ -923,7 +925,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t terrno = pRequest->code; destroyRequest(pRequest); - taos_close(pTscObj); + taos_close_internal(pTscObj); pTscObj = NULL; } else { tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, pTscObj->id, @@ -958,8 +960,8 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { taosMemoryFreeClear(db); connectReq.connType = pObj->connType; - connectReq.pid = htonl(appInfo.pid); - connectReq.startTime = htobe64(appInfo.startTime); + connectReq.pid = appInfo.pid; + connectReq.startTime = appInfo.startTime; tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); tstrncpy(connectReq.user, pObj->user, sizeof(connectReq.user)); @@ -1088,7 +1090,12 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons return NULL; } - return taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY); + STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY); + if (pObj) { + return (TAOS*)pObj->id; + } + + return (TAOS*)0; } TAOS* taos_connect_l(const char* ip, int ipLen, const char* user, int userLen, const char* pass, int passLen, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e3fafa0a82..5f0dbdd415 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -93,10 +93,15 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - return taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); + STscObj* pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); + if (pObj) { + return (TAOS*)pObj->id; + } + + return (TAOS*)0; } -void taos_close(TAOS *taos) { +void taos_close_internal(void *taos) { if (taos == NULL) { return; } @@ -107,6 +112,17 @@ void taos_close(TAOS *taos) { taosRemoveRef(clientConnRefPool, pTscObj->id); } +void taos_close(TAOS *taos) { + STscObj* pObj = acquireTscObj((int64_t)taos); + if (NULL == pObj) { + return; + } + + taos_close_internal(pObj); + releaseTscObj((int64_t)taos); +} + + int taos_errno(TAOS_RES *tres) { if (tres == NULL) { return terrno; @@ -186,29 +202,36 @@ static void syncQueryFn(void* param, void* res, int32_t code) { } TAOS_RES *taos_query(TAOS *taos, const char *sql) { - if (taos == NULL || sql == NULL) { + STscObj* pTscObj = acquireTscObj((int64_t)taos); + if (pTscObj == NULL || sql == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - STscObj* pTscObj = (STscObj*)taos; - #if SYNC_ON_TOP_OF_ASYNC SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); tsem_init(¶m->sem, 0, 0); - taos_query_a(pTscObj, sql, syncQueryFn, param); + taos_query_a(taos, sql, syncQueryFn, param); tsem_wait(¶m->sem); + releaseTscObj((int64_t)taos); + return param->pRequest; #else size_t sqlLen = strlen(sql); if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + releaseTscObj((int64_t)taos); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } - return execQuery(pTscObj, sql, sqlLen); + TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen); + + releaseTscObj((int64_t)taos); + + return pRes; #endif } @@ -425,13 +448,15 @@ int taos_result_precision(TAOS_RES *res) { } int taos_select_db(TAOS *taos, const char *db) { - STscObj *pObj = (STscObj *)taos; + STscObj* pObj = acquireTscObj((int64_t)taos); if (pObj == NULL) { + releaseTscObj((int64_t)taos); terrno = TSDB_CODE_TSC_DISCONNECTED; return TSDB_CODE_TSC_DISCONNECTED; } if (db == NULL || strlen(db) == 0) { + releaseTscObj((int64_t)taos); terrno = TSDB_CODE_TSC_INVALID_INPUT; return terrno; } @@ -443,6 +468,7 @@ int taos_select_db(TAOS *taos, const char *db) { int32_t code = taos_errno(pRequest); taos_free_result(pRequest); + releaseTscObj((int64_t)taos); return code; } @@ -589,19 +615,26 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { int taos_validate_sql(TAOS *taos, const char *sql) { return true; } void taos_reset_current_db(TAOS *taos) { - if (taos == NULL) { + STscObj* pTscObj = acquireTscObj((int64_t)taos); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; return; } - resetConnectDB(taos); + resetConnectDB(pTscObj); + + releaseTscObj((int64_t)taos); } const char *taos_get_server_info(TAOS *taos) { - if (taos == NULL) { + STscObj* pTscObj = acquireTscObj((int64_t)taos); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - STscObj *pTscObj = (STscObj *)taos; + releaseTscObj((int64_t)taos); + return pTscObj->ver; } @@ -619,6 +652,7 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) { if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); + pRequest->stableQuery = pQuery->stableQuery; } if (code == TSDB_CODE_SUCCESS) { @@ -652,10 +686,14 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) { } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { - ASSERT(fp != NULL); - - if (taos == NULL || sql == NULL) { + STscObj* pTscObj = acquireTscObj((int64_t)taos); + if (pTscObj == NULL || sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; + if (pTscObj) { + releaseTscObj((int64_t)taos); + } else { + terrno = TSDB_CODE_TSC_DISCONNECTED; + } fp(param, NULL, terrno); return; } @@ -670,7 +708,7 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param } SRequestObj *pRequest = NULL; - int32_t code = buildRequest(taos, sql, sqlLen, &pRequest); + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; fp(param, NULL, terrno); @@ -866,13 +904,18 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { } TAOS_STMT *taos_stmt_init(TAOS *taos) { - if (taos == NULL) { - tscError("NULL parameter for %s", __FUNCTION__); - terrno = TSDB_CODE_INVALID_PARA; + STscObj* pObj = acquireTscObj((int64_t)taos); + if (NULL == pObj) { + tscError("invalid parameter for %s", __FUNCTION__); + terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - return stmtInit(taos); + TAOS_STMT* pStmt = stmtInit(pObj); + + releaseTscObj((int64_t)taos); + + return pStmt; } int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8c5645e5b6..274ae9984d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1460,7 +1460,7 @@ static void smlDestroyInfo(SSmlHandle* info){ taosMemoryFreeClear(info); } -static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision){ +static SSmlHandle* smlBuildSmlInfo(STscObj* pTscObj, SRequestObj* request, SMLProtocolType protocol, int8_t precision){ int32_t code = TSDB_CODE_SUCCESS; SSmlHandle* info = (SSmlHandle*)taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { @@ -1483,7 +1483,7 @@ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocol } ((SVnodeModifOpStmt*)(info->pQuery->pRoot))->payloadType = PAYLOAD_TYPE_KV; - info->taos = (STscObj *)taos; + info->taos = pTscObj; code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); if(code != TSDB_CODE_SUCCESS){ uError("SML:0x%"PRIx64" get catalog error %d", info->id, code); @@ -2447,13 +2447,21 @@ static void smlInsertCallback(void* param, void* res, int32_t code) { */ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, TSDB_SQL_INSERT); + STscObj* pTscObj = acquireTscObj((int64_t)taos); + if (NULL == pTscObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + uError("SML:taos_schemaless_insert invalid taos"); + return NULL; + } + + SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); if(!request){ + releaseTscObj((int64_t)taos); uError("SML:taos_schemaless_insert error request is null"); return NULL; } - ((STscObj *)taos)->schemalessType = 1; + pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; int cnt = ceil(((double)numLines)/LINE_BATCH); @@ -2468,7 +2476,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr goto end; } - if(isSchemalessDb(((STscObj *)taos), request) != TSDB_CODE_SUCCESS){ + if(isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS){ request->code = TSDB_CODE_SML_INVALID_DB_CONF; smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); goto end; @@ -2493,13 +2501,13 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr } for (int i = 0; i < cnt; ++i) { - SRequestObj* req = (SRequestObj*)createRequest((STscObj *)taos, TSDB_SQL_INSERT); + SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); if(!req){ request->code = TSDB_CODE_OUT_OF_MEMORY; uError("SML:taos_schemaless_insert error request is null"); goto end; } - SSmlHandle* info = smlBuildSmlInfo(taos, req, (SMLProtocolType)protocol, precision); + SSmlHandle* info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); if(!info){ request->code = TSDB_CODE_OUT_OF_MEMORY; uError("SML:taos_schemaless_insert error SSmlHandle is null"); @@ -2533,8 +2541,9 @@ end: taosThreadSpinDestroy(¶ms.lock); tsem_destroy(¶ms.sem); // ((STscObj *)taos)->schemalessType = 0; - ((STscObj *)taos)->schemalessType = 1; + pTscObj->schemalessType = 1; uDebug("resultend:%s", request->msgBuf); + releaseTscObj((int64_t)taos); return (TAOS_RES*)request; } diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7dfb6505e9..1c0caec810 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -478,7 +478,7 @@ int32_t stmtResetStmt(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -TAOS_STMT* stmtInit(TAOS* taos) { +TAOS_STMT* stmtInit(STscObj* taos) { STscObj* pObj = (STscObj*)taos; STscStmt* pStmt = NULL; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 0c35f5691e..98e16a05a6 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -253,8 +253,8 @@ static const SSysTableMeta infosMeta[] = { static const SSysDbTableSchema connectionsSchema[] = { {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, @@ -298,19 +298,18 @@ static const SSysDbTableSchema offsetSchema[] = { }; static const SSysDbTableSchema querySchema[] = { - {.name = "query_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "query_id", .bytes = 26 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, + {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "sql_obj_id", .bytes = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "exec_usec", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, - {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "sub_num", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "sub_status", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 31b8eb71bf..73fd0e7e71 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -210,6 +210,7 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR if (tEncodeI64(pEncoder, desc->stime) < 0) return -1; if (tEncodeI64(pEncoder, desc->reqRid) < 0) return -1; if (tEncodeI32(pEncoder, desc->pid) < 0) return -1; + if (tEncodeI8(pEncoder, desc->stableQuery) < 0) return -1; if (tEncodeCStr(pEncoder, desc->fqdn) < 0) return -1; if (tEncodeI32(pEncoder, desc->subPlanNum) < 0) return -1; @@ -218,7 +219,7 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR for (int32_t m = 0; m < snum; ++m) { SQuerySubDesc *sDesc = taosArrayGet(desc->subDesc, m); if (tEncodeI64(pEncoder, sDesc->tid) < 0) return -1; - if (tEncodeI32(pEncoder, sDesc->status) < 0) return -1; + if (tEncodeCStr(pEncoder, sDesc->status) < 0) return -1; } } } else { @@ -265,6 +266,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) if (tDecodeI64(pDecoder, &desc.stime) < 0) return -1; if (tDecodeI64(pDecoder, &desc.reqRid) < 0) return -1; if (tDecodeI32(pDecoder, &desc.pid) < 0) return -1; + if (tDecodeI8(pDecoder, (int8_t*)&desc.stableQuery) < 0) return -1; if (tDecodeCStrTo(pDecoder, desc.fqdn) < 0) return -1; if (tDecodeI32(pDecoder, &desc.subPlanNum) < 0) return -1; @@ -277,7 +279,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) for (int32_t m = 0; m < snum; ++m) { SQuerySubDesc sDesc = {0}; if (tDecodeI64(pDecoder, &sDesc.tid) < 0) return -1; - if (tDecodeI32(pDecoder, &sDesc.status) < 0) return -1; + if (tDecodeCStrTo(pDecoder, sDesc.status) < 0) return -1; taosArrayPush(desc.subDesc, &sDesc); } } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 2836473793..ba09a182bb 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -39,6 +39,7 @@ typedef struct { int64_t lastAccessTimeMs; uint64_t killId; int32_t numOfQueries; + SRWLatch queryLock; SArray *pQueries; // SArray } SConnObj; @@ -53,8 +54,8 @@ static int32_t mndProcessHeartBeatReq(SRpcMsg *pReq); static int32_t mndProcessConnectReq(SRpcMsg *pReq); static int32_t mndProcessKillQueryReq(SRpcMsg *pReq); static int32_t mndProcessKillConnReq(SRpcMsg *pReq); -static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { @@ -74,7 +75,7 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq); - // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); @@ -129,7 +130,9 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType } static void mndFreeConn(SConnObj *pConn) { + taosWLockLatch(&pConn->queryLock); taosArrayDestroyEx(pConn->pQueries, tFreeClientHbQueryDesc); + taosWUnLockLatch(&pConn->queryLock); mTrace("conn:%u, is destroyed, data:%p", pConn->id, pConn); } @@ -222,8 +225,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto CONN_OVER; } - mndAcquireConn(pMnode, pConn->id); - SConnectRsp connectRsp = {0}; connectRsp.acctId = pUser->acctId; connectRsp.superUser = pUser->superUser; @@ -259,12 +260,17 @@ CONN_OVER: } static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { + taosWLockLatch(&pConn->queryLock); + taosArrayDestroyEx(pConn->pQueries, tFreeClientHbQueryDesc); pConn->pQueries = pBasic->queryDesc; + pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; pBasic->queryDesc = NULL; - pConn->numOfQueries = pBasic->queryDesc ? taosArrayGetSize(pBasic->queryDesc) : 0; + mDebug("queries updated in conn %d, num:%d", pConn->id, pConn->numOfQueries); + + taosWUnLockLatch(&pConn->queryLock); return TSDB_CODE_SUCCESS; } @@ -353,14 +359,10 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb return -1; } else { mDebug("user:%s, conn:%u is freed and create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); + pConn = mndAcquireConn(pMnode, pBasic->connId); } - } else if (pConn->killed) { - mError("user:%s, conn:%u is already killed", connInfo.user, pConn->id); - mndReleaseConn(pMnode, pConn); - terrno = TSDB_CODE_MND_INVALID_CONNECTION; - return -1; } - + SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { mndReleaseConn(pMnode, pConn); @@ -389,6 +391,8 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mndReleaseConn(pMnode, pConn); hbRsp.query = rspBasic; + } else { + mDebug("no query info in hb msg"); } int32_t kvNum = taosHashGetSize(pHbReq->info); @@ -559,14 +563,13 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } } -static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pReq->info.node; - int32_t numOfRows = 0; - SConnObj *pConn = NULL; - int32_t cols = 0; - char *pWrite; - char ipStr[TSDB_IPv4ADDR_LEN + 6]; - +static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SConnObj *pConn = NULL; + if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; pShow->pIter = taosCacheCreateIter(pMgmt->cache); @@ -574,48 +577,45 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, char *data, int3 while (numOfRows < rows) { pConn = mndGetNextConn(pMnode, pShow->pIter); - if (pConn == NULL) break; + if (pConn == NULL) { + pShow->pIter = NULL; + break; + } cols = 0; -#if 0 - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(uint32_t *)pWrite = pConn->id; - cols++; - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->pMeta->pSchemas[cols].bytes); - cols++; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false); - // app name - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->app, pShow->pMeta->pSchemas[cols].bytes); - cols++; + char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(user, pConn->user); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)user, false); - // app pid - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int32_t *)pWrite = pConn->pid; - cols++; + char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; + STR_TO_VARSTR(app, pConn->app); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)app, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - taosIpPort2String(pConn->ip, pConn->port, ipStr); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->pMeta->pSchemas[cols].bytes); - cols++; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConn->pid, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - *(int64_t *)pWrite = pConn->loginTimeMs; - cols++; + char endpoint[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&endpoint[VARSTR_HEADER_SIZE], "%s:%d", taosIpStr(pConn->ip), pConn->port); + varDataLen(endpoint) = strlen(&endpoint[VARSTR_HEADER_SIZE]); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false); - pWrite = data + pShow->offset[cols] * rows + pShow->pMeta->pSchemas[cols].bytes * numOfRows; - if (pConn->lastAccessTimeMs < pConn->loginTimeMs) pConn->lastAccessTimeMs = pConn->loginTimeMs; - *(int64_t *)pWrite = pConn->lastAccessTimeMs; - cols++; -#endif + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConn->loginTimeMs, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pConn->lastAccessTimeMs, false); numOfRows++; } pShow->numOfRows += numOfRows; - return numOfRows; } @@ -633,9 +633,14 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p while (numOfRows < rows) { pConn = mndGetNextConn(pMnode, pShow->pIter); - if (pConn == NULL) break; + if (pConn == NULL) { + pShow->pIter = NULL; + break; + } + taosRLockLatch(&pConn->queryLock); if (NULL == pConn->pQueries || taosArrayGetSize(pConn->pQueries) <= 0) { + taosRUnLockLatch(&pConn->queryLock); continue; } @@ -644,14 +649,28 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p SQueryDesc* pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; + char queryId[26 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&queryId[VARSTR_HEADER_SIZE], "%x:%" PRIx64, pConn->id, pQuery->reqRid); + varDataLen(queryId) = strlen(&queryId[VARSTR_HEADER_SIZE]); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)queryId, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->queryId, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pConn->id, false); + char app[TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE]; + STR_TO_VARSTR(app, pConn->app); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)app, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->pid, false); + char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(b1, pConn->user); + STR_TO_VARSTR(user, pConn->user); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)user, false); @@ -661,23 +680,41 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)endpoint, false); - char b1[9] = {0}; - STR_TO_VARSTR(b1, online ? "ready" : "offline"); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, b1, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->stime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false); - - char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]); + colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->useconds, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, b, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->stableQuery, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false); + + char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t strSize = sizeof(subStatus); + int32_t offset = VARSTR_HEADER_SIZE; + for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { + if (i) { + offset += snprintf(subStatus + offset, strSize - offset - 1, ","); + } + SQuerySubDesc* pDesc = taosArrayGet(pQuery->subDesc, i); + offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); + } + varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, subStatus, false); + + char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(sql, pQuery->sql); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)sql, false); numOfRows++; - sdbRelease(pSdb, pDnode); } + + taosRUnLockLatch(&pConn->queryLock); } pShow->numOfRows += numOfRows; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 02c485fa83..7fffa84e0b 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -468,8 +468,8 @@ int32_t udfdConnectToMnode() { char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)(TSDB_DEFAULT_PASS), strlen(TSDB_DEFAULT_PASS), pass); tstrncpy(connReq.passwd, pass, sizeof(connReq.passwd)); - connReq.pid = htonl(taosGetPId()); - connReq.startTime = htobe64(taosGetTimestampMs()); + connReq.pid = taosGetPId(); + connReq.startTime = taosGetTimestampMs(); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq); void * pReq = rpcMallocCont(contLen); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1ea9124839..6db824e445 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -199,6 +199,7 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_SHOW_SUBSCRIBES_STMT: case QUERY_NODE_SHOW_SMAS_STMT: case QUERY_NODE_SHOW_CONFIGS_STMT: + case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_APPS_STMT: diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index aca7d9c9d3..af22d8f629 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -383,6 +383,16 @@ static int32_t collectMetaKeyFromShowTopics(SCollectMetaKeyCxt* pCxt, SShowStmt* pCxt->pMetaCache); } +static int32_t collectMetaKeyFromShowConnections(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_CONNECTIONS, + pCxt->pMetaCache); +} + +static int32_t collectMetaKeyFromShowQueries(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_QUERIES, + pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromShowTransactions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_TRANS, pCxt->pMetaCache); @@ -453,6 +463,10 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowVgroups(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TOPICS_STMT: return collectMetaKeyFromShowTopics(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_CONNECTIONS_STMT: + return collectMetaKeyFromShowConnections(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_QUERIES_STMT: + return collectMetaKeyFromShowQueries(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return collectMetaKeyFromShowTransactions(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_DELETE_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d623b1a9a9..9e34fe6af7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -42,6 +42,7 @@ typedef struct STranslateContext { SExplainOptions* pExplainOpt; SParseMetaCache* pMetaCache; bool createStream; + bool stableQuery; } STranslateContext; typedef struct SFullDatabaseName { @@ -1478,6 +1479,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_CODE_SUCCESS == code) { code = addNamespace(pCxt, pRealTable); } + if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { + pCxt->stableQuery = true; + } break; } case QUERY_NODE_TEMP_TABLE: { @@ -4054,12 +4058,14 @@ static const char* getSysDbName(ENodeType type) { case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_LICENCE_STMT: case QUERY_NODE_SHOW_CLUSTER_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: return TSDB_INFORMATION_SCHEMA_DB; case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_STREAMS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + case QUERY_NODE_SHOW_APPS_STMT: return TSDB_PERFORMANCE_SCHEMA_DB; default: break; @@ -4109,6 +4115,10 @@ static const char* getSysTableName(ENodeType type) { return TSDB_PERFS_TABLE_TOPICS; case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return TSDB_PERFS_TABLE_TRANS; + case QUERY_NODE_SHOW_VARIABLE_STMT: + return TSDB_INS_TABLE_CONFIGS; + case QUERY_NODE_SHOW_APPS_STMT: + return TSDB_PERFS_TABLE_APPS; default: break; } @@ -4714,6 +4724,7 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code && pClause->ignoreNotExists) { code = TSDB_CODE_SUCCESS; + goto over; } *pIsSuperTable = false; @@ -4807,7 +4818,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { } } - if (isSuperTable) { + if (isSuperTable || 0 == taosHashGetSize(pVgroupHashmap)) { taosHashCleanup(pVgroupHashmap); return TSDB_CODE_SUCCESS; } @@ -5152,6 +5163,8 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_APPS_STMT: code = rewriteShow(pCxt, pQuery); break; case QUERY_NODE_CREATE_TABLE_STMT: @@ -5248,6 +5261,8 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { break; } + pQuery->stableQuery = pCxt->stableQuery; + if (pQuery->haveResultSet) { if (TSDB_CODE_SUCCESS != extractResultSchema(pQuery->pRoot, &pQuery->numOfResCols, &pQuery->pResSchema)) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 18068870d7..0eaeeae9cb 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -151,7 +151,9 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { for (int32_t m = 0; m < pLevel->taskNum; ++m) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); - SQuerySubDesc subDesc = {.tid = pTask->taskId, .status = pTask->status}; + SQuerySubDesc subDesc = {0}; + subDesc.tid = pTask->taskId; + strcpy(subDesc.status, jobTaskStatusStr(pTask->status)); taosArrayPush(pSub, &subDesc); } diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 10a5475555..5f9c2485d1 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -931,15 +931,15 @@ bool taosCacheIterNext(SCacheIter *pIter) { SCacheObj *pCacheObj = pIter->pCacheObj; if (pIter->index + 1 >= pIter->numOfObj) { - if (pIter->entryIndex + 1 >= pCacheObj->capacity) { - return false; - } - // release the reference for all objects in the snapshot for (int32_t i = 0; i < pIter->numOfObj; ++i) { char *p = pIter->pCurrent[i]->data; taosCacheRelease(pCacheObj, (void **)&p, false); pIter->pCurrent[i] = NULL; + } + + if (pIter->entryIndex + 1 >= pCacheObj->capacity) { + return false; } while (1) { From 7f78d4a79b4e4c1ecd78234983b83a14958dbd08 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 15 Jun 2022 19:08:25 +0800 Subject: [PATCH 06/90] feat: partition by tags optimize --- include/libs/nodes/nodes.h | 6 +++ include/libs/nodes/plannodes.h | 3 +- source/libs/executor/src/executorimpl.c | 58 +++++++++++----------- source/libs/nodes/src/nodesCloneFuncs.c | 3 +- source/libs/nodes/src/nodesCodeFuncs.c | 21 +++++--- source/libs/planner/src/planOptimizer.c | 57 ++++++++++++++++++--- source/libs/planner/src/planPhysiCreater.c | 4 +- 7 files changed, 105 insertions(+), 47 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3c25d8add4..3620f45408 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -65,6 +65,12 @@ extern "C" { (list) = NULL; \ } while (0) +#define NODES_CLEAR_LIST(list) \ + do { \ + nodesClearList((list)); \ + (list) = NULL; \ + } while (0) + typedef enum ENodeType { // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN, // VALUE, OPERATOR, FUNCTION and so on. diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 26d055d7d2..749d58b224 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -69,6 +69,7 @@ typedef struct SScanLogicNode { int16_t tsColId; double filesFactor; SArray* pSmaIndexes; + SNodeList* pPartTags; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -257,7 +258,7 @@ typedef struct STableScanPhysiNode { double ratio; int32_t dataRequired; SNodeList* pDynamicScanFuncs; - SNodeList* pPartitionKeys; + SNodeList* pPartitionTags; int64_t interval; int64_t offset; int64_t sliding; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 99a9dcb6af..093cf9417b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2909,7 +2909,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); if (numOfSources == 0) { - qError("%s invalid number: %d of sources in exchange operator", id, (int32_t) numOfSources); + qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); return TSDB_CODE_INVALID_PARA; } @@ -4591,8 +4591,8 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey){ - if(groupKey == NULL) { +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey) { + if (groupKey == NULL) { return TDB_CODE_SUCCESS; } @@ -4601,11 +4601,11 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, return TSDB_CODE_OUT_OF_MEMORY; } int32_t keyLen = 0; - void *keyBuf = NULL; + void* keyBuf = NULL; int32_t numOfGroupCols = taosArrayGetSize(groupKey); for (int32_t j = 0; j < numOfGroupCols; ++j) { SColumn* pCol = taosArrayGet(groupKey, j); - keyLen += pCol->bytes; // actual data + null_flag + keyLen += pCol->bytes; // actual data + null_flag } int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; @@ -4616,9 +4616,9 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, return TSDB_CODE_OUT_OF_MEMORY; } - for(int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++){ - STableKeyInfo *info = taosArrayGet(pTableListInfo->pTableList, i); - SMetaReader mr = {0}; + for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); metaGetTableEntryByUid(&mr, info->uid); @@ -4627,23 +4627,23 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, for (int32_t j = 0; j < numOfGroupCols; ++j) { SColumn* pCol = taosArrayGet(groupKey, j); - if(strcmp(pCol->name, "tbname") == 0){ + if (strcmp(pCol->name, "tbname") == 0) { isNull[i] = 0; memcpy(pStart, mr.me.name, strlen(mr.me.name)); pStart += strlen(mr.me.name); - }else{ + } else { STagVal tagVal = {0}; tagVal.cid = pCol->colId; const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal); - if(p == NULL){ + if (p == NULL) { isNull[j] = 1; continue; } isNull[i] = 0; if (pCol->type == TSDB_DATA_TYPE_JSON) { -// int32_t dataLen = getJsonValueLen(pkey->pData); -// memcpy(pStart, (pkey->pData), dataLen); -// pStart += dataLen; + // int32_t dataLen = getJsonValueLen(pkey->pData); + // memcpy(pStart, (pkey->pData), dataLen); + // pStart += dataLen; } else if (IS_VAR_DATA_TYPE(pCol->type)) { memcpy(pStart, tagVal.pData, tagVal.nData); pStart += tagVal.nData; @@ -4655,7 +4655,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } - int32_t len = (int32_t) (pStart - (char*)keyBuf); + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t* groupId = taosHashGet(pTableListInfo->map, keyBuf, len); if (groupId) { taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), groupId, sizeof(uint64_t)); @@ -4690,16 +4690,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); - code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); //todo for json + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); + code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json taosArrayDestroy(groupKeys); - if (code){ + if (code) { tsdbCleanupReadHandle(pDataReader); return NULL; } - SOperatorInfo* pOperator = - createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; @@ -4726,16 +4725,15 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); - int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); //todo for json + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); + int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json taosArrayDestroy(groupKeys); - if (code){ + if (code) { tsdbCleanupReadHandle(pDataReader); return NULL; } - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4744,7 +4742,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - int32_t code = getTableList(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableListInfo, pScanPhyNode->node.pConditions); + int32_t code = getTableList(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableListInfo, + pScanPhyNode->node.pConditions); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4821,7 +4820,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { - SMergeIntervalPhysiNode * pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; + SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); @@ -5061,7 +5060,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { } SArray* extractPartitionColInfo(SNodeList* pNodeList) { - if(!pNodeList) return NULL; + if (!pNodeList) return NULL; size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); if (pList == NULL) { @@ -5166,7 +5165,7 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa SArray* res = taosArrayInit(8, sizeof(uint64_t)); code = doFilterTag(pTagCond, &metaArg, res); - if (code == TSDB_CODE_INDEX_REBUILDING){ // todo + if (code == TSDB_CODE_INDEX_REBUILDING) { // todo // doFilter(); } else if (code != TSDB_CODE_SUCCESS) { qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid); @@ -5524,4 +5523,3 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF } return code; } - diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1ef0ccf7f9..2e30b01357 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -355,6 +355,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(tsColId); COPY_SCALAR_FIELD(filesFactor); + CLONE_NODE_LIST_FIELD(pPartTags); return (SNode*)pDst; } @@ -495,7 +496,7 @@ static SNode* physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhys COPY_SCALAR_FIELD(ratio); COPY_SCALAR_FIELD(dataRequired); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); - CLONE_NODE_LIST_FIELD(pPartitionKeys); + CLONE_NODE_LIST_FIELD(pPartitionTags); COPY_SCALAR_FIELD(interval); COPY_SCALAR_FIELD(offset); COPY_SCALAR_FIELD(sliding); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 968bb75997..fb6a428d3c 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -515,6 +515,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; static const char* jkScanLogicPlanTableId = "TableId"; static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTagCond = "TagCond"; +static const char* jkScanLogicPlanPartTags = "PartTags"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; @@ -535,6 +536,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags); + } return code; } @@ -559,6 +563,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags); + } return code; } @@ -1368,6 +1375,7 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType"; static const char* jkTableScanPhysiPlanWatermark = "watermark"; static const char* jkTableScanPhysiPlanTsColId = "tsColId"; static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor"; +static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags"; static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; @@ -1421,6 +1429,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags); + } return code; } @@ -1446,30 +1457,24 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanDataRequired, pNode->dataRequired, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkTableScanPhysiPlanDynamicScanFuncs, &pNode->pDynamicScanFuncs); } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanInterval, pNode->interval, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanOffset, pNode->offset, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSliding, pNode->sliding, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanIntervalUnit, pNode->intervalUnit, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanSlidingUnit, pNode->slidingUnit, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableScanPhysiPlanTriggerType, pNode->triggerType, code); @@ -1483,6 +1488,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags); + } + return code; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 41b80eaaa8..9d7cd0cf27 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -109,9 +109,8 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { return false; } if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) || - (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && - pNode->pParent->pParent && - QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent)) ) { + (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && pNode->pParent->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent))) { return true; } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); @@ -222,9 +221,8 @@ static int32_t osdGetDataRequired(SNodeList* pFuncs) { static void setScanWindowInfo(SScanLogicNode* pScan) { SLogicNode* pParent = pScan->node.pParent; - if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && - pParent->pParent && - QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pParent) && pParent->pParent && + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent->pParent)) { pParent = pParent->pParent; } if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pParent)) { @@ -1041,12 +1039,55 @@ static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) return smaOptimizeImpl(pCxt, pLogicSubplan, pScan); } +static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType) { + *(bool*)pContext = true; + return DEAL_RES_END; + } + } + return DEAL_RES_CONTINUE; +} + +static bool partTagsOptHasCol(SNodeList* pPartKeys) { + bool hasCol = false; + nodesWalkExprs(pPartKeys, partTagsOptHasColImpl, &hasCol); + return hasCol; +} + +static bool partTagsOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + return false; + } + + return !partTagsOptHasCol(((SPartitionLogicNode*)pNode)->pPartitionKeys); +} + +static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SPartitionLogicNode* pPart = + (SPartitionLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); + if (NULL == pPart) { + return TSDB_CODE_SUCCESS; + } + + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pPart->node.pChildren, 0); + TSWAP(pPart->pPartitionKeys, pScan->pPartTags); + int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pPart, (SLogicNode*)pScan); + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pPart->node.pChildren); + nodesDestroyNode((SNode*)pPart); + } + return code; +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { - {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, + {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, - {.pName = "SmaIndex", .optimizeFunc = smaOptimize} + {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, + {.pName = "PartitionByTags", .optimizeFunc = partTagsOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 4a0348151b..1b974af637 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -500,7 +500,9 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); pTableScan->dataRequired = pScanLogicNode->dataRequired; pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs); - if (NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) { + pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags); + if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) || + (NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) { nodesDestroyNode((SNode*)pTableScan); return TSDB_CODE_OUT_OF_MEMORY; } From f93bd44f97cbaa645dbbf6705eafa83e76b62d28 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 20:29:39 +0800 Subject: [PATCH 07/90] fix ut issue and crash issue --- source/dnode/mnode/impl/src/mndProfile.c | 1 - source/dnode/mnode/impl/test/profile/profile.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ba09a182bb..20d94e341c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -359,7 +359,6 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb return -1; } else { mDebug("user:%s, conn:%u is freed and create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); - pConn = mndAcquireConn(pMnode, pBasic->connId); } } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 794374a91d..039700ae9e 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -88,7 +88,7 @@ TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) { TEST_F(MndTestProfile, 03_ConnectMsg_Show) { test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", ""); - EXPECT_EQ(test.GetShowRows(), 0); + EXPECT_EQ(test.GetShowRows(), 1); } TEST_F(MndTestProfile, 04_HeartBeatMsg) { From b0f048a3f2d539fc54003f1820cc3595b863fa3b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 15 Jun 2022 20:57:55 +0800 Subject: [PATCH 08/90] feat: redistribute vgroup --- source/dnode/mnode/impl/src/mndTrans.c | 19 +++++++++++++++---- source/dnode/vnode/src/vnd/vnodeSync.c | 14 +++++++++++++- ...distribute_vgroup_replica3_v1_follower.sim | 19 +++++++++++-------- 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 4be40e6a5c..5b819ab092 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1078,6 +1078,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (numOfActions == 0) return code; if (pTrans->redoActionPos >= numOfActions) return code; + int32_t retryTimes = 0; + for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); @@ -1126,17 +1128,26 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) pTrans->code = terrno; mError("trans:%d, %s:%d is executed and failed to sync to other mnodes since %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, terrstr()); - break; + if (retryTimes >= 3) + break; + else + continue; } + retryTimes = 0; } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; } else { terrno = code; pTrans->code = code; - mError("trans:%d, %s:%d failed to execute since %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, - terrstr()); - break; + pAction->epSet.inUse++; + if (pAction->epSet.inUse >= pAction->epSet.numOfEps) pAction->epSet.inUse = 0; + mError("trans:%d, %s:%d failed to execute since %s, inUse set to %d", pTrans->id, mndTransStr(pAction->stage), + pAction->id, terrstr(), pAction->epSet.inUse); + if (retryTimes >= 3) + break; + else + continue; } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 975dff9fb6..ce0c21bc35 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -72,7 +72,19 @@ static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) { return -1; } - return syncPropose(pVnode->sync, &rpcMsg, false); + int32_t code = syncPropose(pVnode->sync, &rpcMsg, false); + if (code != 0) { + vDebug("vgId:%d, failed to propose reconfig msg since %s", TD_VID(pVnode), terrstr()); + if (syncLeaderTransfer(pVnode->sync) != 0) { + vError("vgId:%d, failed to transfer leader since %s", TD_VID(pVnode), terrstr()); + } else { + vDebug("vgId:%d, transfer leader success, propose reconfig config again", TD_VID(pVnode)); + taosMsleep(10); + code = syncPropose(pVnode->sync, &rpcMsg, false); + } + } + + return code; } void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 73be4a35bf..3532332174 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -4,6 +4,11 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start @@ -128,9 +133,9 @@ if $rows != 1 then endi if $data(2)[4] == leader then $leaderExist = 1 - $leaderVnode = 4 - $follower1 = 2 - $follower2 = 3 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 endi if $data(2)[6] == leader then $leaderExist = 1 @@ -140,9 +145,9 @@ if $data(2)[6] == leader then endi if $data(2)[8] == leader then $leaderExist = 1 - $leaderVnode = 2 - $follower1 = 3 - $follower2 = 4 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 endi if $leaderExist != 1 then goto step3 @@ -171,8 +176,6 @@ if $rows != 1 then return -1 endi -return - print =============== step33: move follower1 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 From b56f1dc03de7834fc177254653882ed0bfc0ae7e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 20:58:18 +0800 Subject: [PATCH 09/90] fix schemaless taos issue --- source/client/src/clientSml.c | 10 +++++----- source/dnode/mnode/impl/src/mndStb.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 2ec6c2539a..25018e0d15 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -309,7 +309,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_ADD_COLUMN: { int n = sprintf(result, "alter stable `%s` add column ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); const char *errStr = taos_errstr(res); if (code != TSDB_CODE_SUCCESS) { @@ -323,7 +323,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_ADD_TAG: { int n = sprintf(result, "alter stable `%s` add tag ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); const char *errStr = taos_errstr(res); if (code != TSDB_CODE_SUCCESS) { @@ -337,7 +337,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { int n = sprintf(result, "alter stable `%s` modify column ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); @@ -350,7 +350,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_CHANGE_TAG_SIZE: { int n = sprintf(result, "alter stable `%s` modify tag ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); @@ -405,7 +405,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); - TAOS_RES *res = taos_query(info->taos, result); + TAOS_RES *res = taos_query((TAOS*)info->taos->id, result); code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3e91bfa926..3e21a1f942 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1287,7 +1287,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_STB; + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } From 4973739794d4bdc10460f3c50a94596015ab2753 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 15 Jun 2022 20:59:33 +0800 Subject: [PATCH 10/90] show apps --- include/common/tglobal.h | 1 + include/common/tmsg.h | 26 ++- source/client/inc/clientInt.h | 33 +-- source/client/src/clientEnv.c | 4 +- source/client/src/clientHb.c | 69 +++++- source/common/src/systable.c | 19 ++ source/common/src/tglobal.c | 5 +- source/common/src/tmsg.c | 34 ++- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 2 + source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 23 ++ source/dnode/mnode/impl/inc/mndInt.h | 3 +- source/dnode/mnode/impl/src/mndProfile.c | 240 ++++++++++++++++++-- source/dnode/mnode/impl/src/mndShow.c | 2 + source/dnode/mnode/impl/src/mndStb.c | 3 +- source/libs/catalog/src/ctgCache.c | 2 - 15 files changed, 400 insertions(+), 66 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1b44b6d7ea..8c03d3ff42 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -61,6 +61,7 @@ extern int32_t tsNumOfRpcThreads; extern int32_t tsNumOfCommitThreads; extern int32_t tsNumOfTaskQueueThreads; extern int32_t tsNumOfMnodeQueryThreads; +extern int32_t tsNumOfMnodeFetchThreads; extern int32_t tsNumOfMnodeReadThreads; extern int32_t tsNumOfVnodeQueryThreads; extern int32_t tsNumOfVnodeFetchThreads; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 694a3cdbdf..27ff27a598 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -106,6 +106,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_CONNS, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_VNODES, + TSDB_MGMT_TABLE_APPS, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -2000,7 +2001,6 @@ typedef struct { int64_t useconds; int64_t stime; // timestamp precision ms int64_t reqRid; - int32_t pid; bool stableQuery; char fqdn[TSDB_FQDN_LEN]; int32_t subPlanNum; @@ -2009,8 +2009,6 @@ typedef struct { typedef struct { uint32_t connId; - int32_t pid; - char app[TSDB_APP_NAME_LEN]; SArray* queryDesc; // SArray } SQueryHbReqBasic; @@ -2025,9 +2023,31 @@ typedef struct { SArray* pQnodeList; } SQueryHbRspBasic; +typedef struct SAppClusterSummary { + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj +} SAppClusterSummary; + +typedef struct { + int64_t appId; + int32_t pid; + char name[TSDB_APP_NAME_LEN]; + int64_t startTime; + SAppClusterSummary summary; +} SAppHbReq; + typedef struct { SClientHbKey connKey; int64_t clusterId; + SAppHbReq app; SQueryHbReqBasic* query; SHashObj* info; // hash } SClientHbReq; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 69a24e20cd..98326ae830 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -76,10 +76,12 @@ typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq typedef struct { int8_t inited; + int64_t appId; // ctl int8_t threadStop; TdThread thread; TdThreadMutex lock; // used when app init and cleanup + SHashObj *appSummary; SArray* appHbMgrs; // SArray one for each cluster FHbReqHandle reqHandle[CONN_TYPE__MAX]; FHbRspHandle rspHandle[CONN_TYPE__MAX]; @@ -92,33 +94,20 @@ typedef struct SQueryExecMetric { int64_t rsp; // receive response from server, us } SQueryExecMetric; -typedef struct SInstanceSummary { - uint64_t numOfInsertsReq; - uint64_t numOfInsertRows; - uint64_t insertElapsedTime; - uint64_t insertBytes; // submit to tsdb since launched. - - uint64_t fetchBytes; - uint64_t queryElapsedTime; - uint64_t numOfSlowQueries; - uint64_t totalRequests; - uint64_t currentRequests; // the number of SRequestObj -} SInstanceSummary; - typedef struct SHeartBeatInfo { void* pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - TdThreadMutex qnodeMutex; - SArray* pQnodeList; - SInstanceSummary summary; - SList* pConnList; // STscObj linked list - uint64_t clusterId; - void* pTransporter; - SAppHbMgr* pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + TdThreadMutex qnodeMutex; + SArray* pQnodeList; + SAppClusterSummary summary; + SList* pConnList; // STscObj linked list + uint64_t clusterId; + void* pTransporter; + SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb60624145..3b74554e76 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -48,7 +48,7 @@ static void registerRequest(SRequestObj *pRequest) { int32_t num = atomic_add_fetch_32(&pTscObj->numOfReqs, 1); if (pTscObj->pAppInfo) { - SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary; + SAppClusterSummary *pSummary = &pTscObj->pAppInfo->summary; int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); @@ -62,7 +62,7 @@ static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); STscObj *pTscObj = pRequest->pTscObj; - SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; + SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2365e6e259..72b7988f25 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -314,7 +314,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.queryId = pRequest->requestId; desc.useconds = now - pRequest->metric.start; desc.reqRid = pRequest->self; - desc.pid = hbBasic->pid; desc.stableQuery = pRequest->stableQuery; taosGetFqdn(desc.fqdn); desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0; @@ -360,8 +359,6 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { } hbBasic->connId = pTscObj->connId; - hbBasic->pid = taosGetPId(); - taosGetAppName(hbBasic->app, NULL); int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; if (numOfQueries <= 0) { @@ -507,6 +504,21 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC return TSDB_CODE_SUCCESS; } +int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { + SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + if (NULL != pApp) { + memcpy(&req->app, pApp, sizeof(*pApp)); + } else { + memset(&req->app.summary, 0, sizeof(req->app.summary)); + req->app.pid = taosGetPId(); + req->app.appId = clientHbMgr.appId; + taosGetAppName(req->app.name, NULL); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -517,6 +529,8 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req return code; } + hbGetAppInfo(*clusterId, req); + hbGetQueryBasicInfo(connKey, req); code = hbGetExpiredUserInfo(connKey, pCatalog, req); @@ -589,6 +603,47 @@ void hbThreadFuncUnexpectedStopped(void) { atomic_store_8(&clientHbMgr.threadStop, 2); } +void hbMergeSummary(SAppClusterSummary* dst, SAppClusterSummary* src) { + dst->numOfInsertsReq += src->numOfInsertsReq; + dst->numOfInsertRows += src->numOfInsertRows; + dst->insertElapsedTime += src->insertElapsedTime; + dst->insertBytes += src->insertBytes; + dst->fetchBytes += src->fetchBytes; + dst->queryElapsedTime += src->queryElapsedTime; + dst->numOfSlowQueries += src->numOfSlowQueries; + dst->totalRequests += src->totalRequests; + dst->currentRequests += src->currentRequests; +} + +int32_t hbGatherAppInfo(void) { + SAppHbReq req = {0}; + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + if (sz > 0) { + req.pid = taosGetPId(); + req.appId = clientHbMgr.appId; + taosGetAppName(req.name, NULL); + } + + for (int32_t i = 0; i < sz; ++i) { + SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; + SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); + if (NULL == pApp) { + memcpy(&req.summary, &pAppHbMgr->pAppInstInfo->summary, sizeof(req.summary)); + taosHashPut(clientHbMgr.appSummary, &clusterId, sizeof(clusterId), &req, sizeof(req)); + } else { + if (pAppHbMgr->startTime < pApp->startTime) { + pApp->startTime = pAppHbMgr->startTime; + } + + hbMergeSummary(&pApp->summary, &pAppHbMgr->pAppInstInfo->summary); + } + } + + return TSDB_CODE_SUCCESS; +} + + static void *hbThreadFunc(void *param) { setThreadName("hb"); #ifdef WINDOWS @@ -603,6 +658,10 @@ static void *hbThreadFunc(void *param) { taosThreadMutexLock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); + if (sz > 0) { + hbGatherAppInfo(); + } + for (int i = 0; i < sz; i++) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -746,6 +805,10 @@ int hbMgrInit() { int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; + clientHbMgr.appId = tGenIdPI64(); + tscDebug("app %" PRIx64 " initialized", clientHbMgr.appId); + + clientHbMgr.appSummary = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); taosThreadMutexInit(&clientHbMgr.lock, NULL); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cb89784faf..21161b8444 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -315,6 +315,24 @@ static const SSysDbTableSchema querySchema[] = { {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; +static const SSysDbTableSchema appSchema[] = { + {.name = "app_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "start_time", .bytes = 8 , .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "insert_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "insert_row", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "insert_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "insert_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "fetch_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "query_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "show_query", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "total_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "current_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, +}; + static const SSysTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)}, {TSDB_PERFS_TABLE_QUERIES, querySchema, tListLen(querySchema)}, @@ -325,6 +343,7 @@ static const SSysTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)}, {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, + {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)} }; void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fbb4f78425..2e1a539980 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -51,10 +51,11 @@ int32_t tsNumOfShmThreads = 1; int32_t tsNumOfRpcThreads = 1; int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 1; -int32_t tsNumOfMnodeQueryThreads = 1; +int32_t tsNumOfMnodeQueryThreads = 2; +int32_t tsNumOfMnodeFetchThreads = 1; int32_t tsNumOfMnodeReadThreads = 1; int32_t tsNumOfVnodeQueryThreads = 2; -int32_t tsNumOfVnodeFetchThreads = 2; +int32_t tsNumOfVnodeFetchThreads = 1; int32_t tsNumOfVnodeWriteThreads = 2; int32_t tsNumOfVnodeSyncThreads = 2; int32_t tsNumOfVnodeMergeThreads = 2; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 923e59fbe8..3fc72e047c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -191,13 +191,25 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1; if (pReq->connKey.connType == CONN_TYPE__QUERY) { + if (tEncodeI64(pEncoder, pReq->app.appId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->app.pid) < 0) return -1; + if (tEncodeCStr(pEncoder, pReq->app.name) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->app.startTime) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.numOfInsertsReq) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.numOfInsertRows) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.insertElapsedTime) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.insertBytes) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.fetchBytes) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.queryElapsedTime) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.numOfSlowQueries) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.totalRequests) < 0) return -1; + if (tEncodeU64(pEncoder, pReq->app.summary.currentRequests) < 0) return -1; + int32_t queryNum = 0; if (pReq->query) { queryNum = 1; if (tEncodeI32(pEncoder, queryNum) < 0) return -1; if (tEncodeU32(pEncoder, pReq->query->connId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->query->pid) < 0) return -1; - if (tEncodeCStr(pEncoder, pReq->query->app) < 0) return -1; int32_t num = taosArrayGetSize(pReq->query->queryDesc); if (tEncodeI32(pEncoder, num) < 0) return -1; @@ -209,7 +221,6 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR if (tEncodeI64(pEncoder, desc->useconds) < 0) return -1; if (tEncodeI64(pEncoder, desc->stime) < 0) return -1; if (tEncodeI64(pEncoder, desc->reqRid) < 0) return -1; - if (tEncodeI32(pEncoder, desc->pid) < 0) return -1; if (tEncodeI8(pEncoder, desc->stableQuery) < 0) return -1; if (tEncodeCStr(pEncoder, desc->fqdn) < 0) return -1; if (tEncodeI32(pEncoder, desc->subPlanNum) < 0) return -1; @@ -243,14 +254,26 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) if (tDecodeSClientHbKey(pDecoder, &pReq->connKey) < 0) return -1; if (pReq->connKey.connType == CONN_TYPE__QUERY) { + if (tDecodeI64(pDecoder, &pReq->app.appId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->app.pid) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pReq->app.name) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->app.startTime) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.numOfInsertsReq) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.numOfInsertRows) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.insertElapsedTime) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.insertBytes) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.fetchBytes) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.queryElapsedTime) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.numOfSlowQueries) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.totalRequests) < 0) return -1; + if (tDecodeU64(pDecoder, &pReq->app.summary.currentRequests) < 0) return -1; + int32_t queryNum = 0; if (tDecodeI32(pDecoder, &queryNum) < 0) return -1; if (queryNum) { pReq->query = taosMemoryCalloc(1, sizeof(*pReq->query)); if (NULL == pReq->query) return -1; if (tDecodeU32(pDecoder, &pReq->query->connId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->query->pid) < 0) return -1; - if (tDecodeCStrTo(pDecoder, pReq->query->app) < 0) return -1; int32_t num = 0; if (tDecodeI32(pDecoder, &num) < 0) return -1; @@ -265,7 +288,6 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) if (tDecodeI64(pDecoder, &desc.useconds) < 0) return -1; if (tDecodeI64(pDecoder, &desc.stime) < 0) return -1; if (tDecodeI64(pDecoder, &desc.reqRid) < 0) return -1; - if (tDecodeI32(pDecoder, &desc.pid) < 0) return -1; if (tDecodeI8(pDecoder, (int8_t*)&desc.stableQuery) < 0) return -1; if (tDecodeCStrTo(pDecoder, desc.fqdn) < 0) return -1; if (tDecodeI32(pDecoder, &desc.subPlanNum) < 0) return -1; diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index c5c3d76f1e..a4c37dd334 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -30,6 +30,7 @@ typedef struct SMnodeMgmt { const char *path; const char *name; SSingleWorker queryWorker; + SSingleWorker fetchWorker; SSingleWorker readWorker; SSingleWorker writeWorker; SSingleWorker syncWorker; @@ -57,6 +58,7 @@ int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 7cd7da1aa9..5723c77cfd 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -122,6 +122,13 @@ int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } +int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + pMsg->info.node = pMgmt->pMnode; + + return mmPutMsgToWorker(pMgmt, &pMgmt->fetchWorker, pMsg); +} + + int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } @@ -135,6 +142,9 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { case QUERY_QUEUE: pWorker = &pMgmt->queryWorker; break; + case FETCH_QUEUE: + pWorker = &pMgmt->fetchWorker; + break; case READ_QUEUE: pWorker = &pMgmt->readWorker; break; @@ -167,6 +177,18 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { return -1; } + SSingleWorkerCfg fCfg = { + .min = tsNumOfMnodeFetchThreads, + .max = tsNumOfMnodeFetchThreads, + .name = "mnode-fetch", + .fp = (FItem)mmProcessRpcMsg, + .param = pMgmt, + }; + if (tSingleWorkerInit(&pMgmt->fetchWorker, &fCfg) != 0) { + dError("failed to start mnode-fetch worker since %s", terrstr()); + return -1; + } + SSingleWorkerCfg rCfg = { .min = tsNumOfMnodeReadThreads, .max = tsNumOfMnodeReadThreads, @@ -227,6 +249,7 @@ void mmStopWorker(SMnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->monitorWorker); tSingleWorkerCleanup(&pMgmt->queryWorker); + tSingleWorkerCleanup(&pMgmt->fetchWorker); tSingleWorkerCleanup(&pMgmt->readWorker); tSingleWorkerCleanup(&pMgmt->writeWorker); tSingleWorkerCleanup(&pMgmt->syncWorker); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index cc9bc5b634..37bae1d5c0 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -67,7 +67,8 @@ typedef struct { } SShowMgmt; typedef struct { - SCacheObj *cache; + SCacheObj *connCache; + SCacheObj *appCache; } SProfileMgmt; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ba09a182bb..73934f5144 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -43,6 +43,16 @@ typedef struct { SArray *pQueries; // SArray } SConnObj; +typedef struct { + int64_t appId; + uint32_t ip; + int32_t pid; + char name[TSDB_APP_NAME_LEN]; + int64_t startTime; + SAppClusterSummary summary; + int64_t lastAccessTimeMs; +} SAppObj; + static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); @@ -57,14 +67,24 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq); static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); +static void mndFreeApp(SAppObj *pApp); +static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static void mndCancelGetNextApp(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; // in ms - int32_t connCheckTime = tsShellActivityTimer * 2 * 1000; - pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); - if (pMgmt->cache == NULL) { + int32_t checkTime = tsShellActivityTimer * 2 * 1000; + pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_INT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + if (pMgmt->connCache == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to alloc profile cache since %s", terrstr()); + return -1; + } + + pMgmt->appCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, checkTime, true, (__cache_free_fn_t)mndFreeApp, "app"); + if (pMgmt->appCache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc profile cache since %s", terrstr()); return -1; @@ -79,15 +99,22 @@ int32_t mndInitProfile(SMnode *pMnode) { mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndRetrieveApps); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_APPS, mndCancelGetNextApp); return 0; } void mndCleanupProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - if (pMgmt->cache != NULL) { - taosCacheCleanup(pMgmt->cache); - pMgmt->cache = NULL; + if (pMgmt->connCache != NULL) { + taosCacheCleanup(pMgmt->connCache); + pMgmt->connCache = NULL; + } + + if (pMgmt->appCache != NULL) { + taosCacheCleanup(pMgmt->appCache); + pMgmt->appCache = NULL; } } @@ -118,7 +145,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); int32_t keepTime = tsShellActivityTimer * 3; - SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); + SConnObj *pConn = taosCachePut(pMgmt->connCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); @@ -140,14 +167,13 @@ static void mndFreeConn(SConnObj *pConn) { static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(connId)); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(connId)); if (pConn == NULL) { mDebug("conn:%u, already destroyed", connId); return NULL; } - int32_t keepTime = tsShellActivityTimer * 3; - pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); + pConn->lastAccessTimeMs = taosGetTimestampMs(); mTrace("conn:%u, acquired from cache, data:%p", pConn->id, pConn); return pConn; @@ -158,7 +184,7 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { mTrace("conn:%u, released from cache, data:%p", pConn->id, pConn); SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); } void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) { @@ -275,6 +301,77 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { return TSDB_CODE_SUCCESS; } +static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq* pReq) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SAppObj app; + app.appId = pReq->appId; + app.ip = clientIp; + app.pid = pReq->pid; + strcpy(app.name, pReq->name); + app.startTime = pReq->startTime; + memcpy(&app.summary, &pReq->summary, sizeof(pReq->summary)); + app.lastAccessTimeMs = taosGetTimestampMs(); + + int32_t keepTime = tsShellActivityTimer * 3; + SAppObj *pApp = taosCachePut(pMgmt->appCache, &pReq->appId, sizeof(pReq->appId), &app, sizeof(app), keepTime * 1000); + if (pApp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to app %" PRIx64 " into cache since %s", pReq->appId, terrstr()); + return NULL; + } + + mTrace("app %" PRIx64 " is put into cache", pReq->appId); + return pApp; +} + +static void mndFreeApp(SAppObj *pApp) { + mTrace("app %" PRIx64 " is destroyed", pApp->appId); +} + + +static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SAppObj *pApp = taosCacheAcquireByKey(pMgmt->appCache, &appId, sizeof(appId)); + if (pApp == NULL) { + mDebug("app %" PRIx64 " not in cache", appId); + return NULL; + } + + pApp->lastAccessTimeMs = (uint64_t)taosGetTimestampMs(); + + mTrace("app %" PRIx64 " acquired from cache", appId); + return pApp; +} + +static void mndReleaseApp(SMnode *pMnode, SAppObj *pApp) { + if (pApp == NULL) return; + mTrace("release app %" PRIx64 " to cache", pApp->appId); + + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosCacheRelease(pMgmt->appCache, (void **)&pApp, false); +} + +void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { + SAppObj *pApp = NULL; + bool hasNext = taosCacheIterNext(pIter); + if (hasNext) { + size_t dataLen = 0; + pApp = taosCacheIterGetData(pIter, &dataLen); + } else { + taosCacheDestroyIter(pIter); + } + + return pApp; +} + +static void mndCancelGetNextApp(SMnode *pMnode, void *pIter) { + if (pIter != NULL) { + taosCacheDestroyIter(pIter); + } +} + static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { #if 0 SClientHbRsp* pRsp = taosMemoryMalloc(sizeof(SClientHbRsp)); @@ -340,26 +437,47 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { return NULL; } +static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) { + SAppHbReq* pReq = &pHbReq->app; + SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId); + if (pApp == NULL) { + pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq); + if (pApp == NULL) { + mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr()); + return -1; + } else { + mDebug("a new app %" PRIx64 "created", pReq->appId); + return TSDB_CODE_SUCCESS; + } + } + + memcpy(&pApp->summary, &pReq->summary, sizeof(pReq->summary)); + + mndReleaseApp(pMnode, pApp); + + return TSDB_CODE_SUCCESS; +} + static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHbReq *pHbReq, SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; + SRpcConnInfo connInfo = pMsg->conn; + + mndUpdateAppInfo(pMnode, pHbReq, &connInfo); if (pHbReq->query) { SQueryHbReqBasic *pBasic = pHbReq->query; - SRpcConnInfo connInfo = pMsg->conn; - SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId); if (pConn == NULL) { pConn = mndCreateConn(pMnode, connInfo.user, CONN_TYPE__QUERY, connInfo.clientIp, connInfo.clientPort, - pBasic->pid, pBasic->app, 0); + pHbReq->app.pid, pHbReq->app.name, 0); if (pConn == NULL) { mError("user:%s, conn:%u is freed and failed to create new since %s", connInfo.user, pBasic->connId, terrstr()); return -1; } else { - mDebug("user:%s, conn:%u is freed and create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); - pConn = mndAcquireConn(pMnode, pBasic->connId); + mDebug("user:%s, conn:%u is freed, will create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); } } @@ -518,7 +636,7 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { mInfo("kill query msg is received, queryId:%d", killReq.queryId); - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &killReq.connId, sizeof(int32_t)); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(int32_t)); if (pConn == NULL) { mError("connId:%d, failed to kill queryId:%d, conn not exist", killReq.connId, killReq.queryId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; @@ -526,7 +644,7 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } else { mInfo("connId:%d, queryId:%d is killed by user:%s", killReq.connId, killReq.queryId, pReq->conn.user); pConn->killId = killReq.queryId; - taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); return 0; } } @@ -550,7 +668,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { return -1; } - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &killReq.connId, sizeof(int32_t)); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(int32_t)); if (pConn == NULL) { mError("connId:%d, failed to kill connection, conn not exist", killReq.connId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; @@ -558,7 +676,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } else { mInfo("connId:%d, is killed by user:%s", killReq.connId, pReq->conn.user); pConn->killed = 1; - taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; } } @@ -572,7 +690,7 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - pShow->pIter = taosCacheCreateIter(pMgmt->cache); + pShow->pIter = taosCacheCreateIter(pMgmt->connCache); } while (numOfRows < rows) { @@ -628,7 +746,7 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - pShow->pIter = taosCacheCreateIter(pMgmt->cache); + pShow->pIter = taosCacheCreateIter(pMgmt->connCache); } while (numOfRows < rows) { @@ -667,7 +785,7 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p colDataAppend(pColInfo, numOfRows, (const char *)app, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->pid, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pConn->pid, false); char user[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(user, pConn->user); @@ -721,6 +839,80 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p return numOfRows; } +static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SAppObj *pApp = NULL; + + if (pShow->pIter == NULL) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + pShow->pIter = taosCacheCreateIter(pMgmt->appCache); + } + + while (numOfRows < rows) { + pApp = mndGetNextApp(pMnode, pShow->pIter); + if (pApp == NULL) { + pShow->pIter = NULL; + break; + } + + cols = 0; + + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->appId, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->ip, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->pid, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->name, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->startTime, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.numOfInsertsReq, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.numOfInsertRows, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.insertElapsedTime, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.insertBytes, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.fetchBytes, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.queryElapsedTime, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.numOfSlowQueries, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.totalRequests, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.currentRequests, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)pApp->lastAccessTimeMs, false); + + numOfRows++; + } + + pShow->numOfRows += numOfRows; + return numOfRows; +} + + static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { if (pIter != NULL) { taosCacheDestroyIter(pIter); @@ -729,5 +921,5 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { int32_t mndGetNumOfConnections(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - return taosCacheGetNumOfObj(pMgmt->cache); + return taosCacheGetNumOfObj(pMgmt->connCache); } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6e569a04cc..d9274d54ae 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -103,6 +103,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_TOPICS; } else if (strncasecmp(name, TSDB_PERFS_TABLE_STREAMS, len) == 0) { type = TSDB_MGMT_TABLE_STREAMS; + } else if (strncasecmp(name, TSDB_PERFS_TABLE_APPS, len) == 0) { + type = TSDB_MGMT_TABLE_APPS; } else { // ASSERT(0); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3e91bfa926..88a857f599 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1287,7 +1287,8 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_STB; + //terrno = TSDB_CODE_MND_INVALID_STB; + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; return -1; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 62890b8326..d2fd72bdc4 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1972,8 +1972,6 @@ void* ctgUpdateThreadFunc(void* param) { ctgdShowClusterCache(pCtg); } - if (CTG_IS_LOCKED(&gCtgMgmt.lock)) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); - qInfo("catalog update thread stopped"); return NULL; From f2de1af904765db5a6e8e08eca2485ba02f703a2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 15 Jun 2022 23:00:31 +0800 Subject: [PATCH 11/90] enh(query): partition by support arithmetic expr, and do some internal refactor. --- include/libs/function/function.h | 4 - source/libs/executor/inc/executil.h | 1 - source/libs/executor/inc/executorimpl.h | 26 ++-- source/libs/executor/src/executil.c | 11 -- source/libs/executor/src/executorimpl.c | 133 +----------------- source/libs/executor/src/groupoperator.c | 49 +++++-- source/libs/executor/src/scanoperator.c | 11 +- source/libs/executor/src/timewindowoperator.c | 5 - source/libs/function/inc/texpr.h | 67 --------- source/libs/function/src/texpr.c | 8 -- 10 files changed, 59 insertions(+), 256 deletions(-) delete mode 100644 source/libs/function/inc/texpr.h diff --git a/include/libs/function/function.h b/include/libs/function/function.h index b359fa5d6a..54762a0f17 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -150,12 +150,8 @@ typedef struct SqlFunctionCtx { } SqlFunctionCtx; enum { - TEXPR_NODE_DUMMY = 0x0, TEXPR_BINARYEXPR_NODE= 0x1, TEXPR_UNARYEXPR_NODE = 0x2, - TEXPR_FUNCTION_NODE = 0x3, - TEXPR_COL_NODE = 0x4, - TEXPR_VALUE_NODE = 0x8, }; typedef struct tExprNode { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index ca25b7aab1..a70cff5552 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -55,7 +55,6 @@ typedef struct SResultRow { uint32_t numOfRows; // number of rows of current time window STimeWindow win; struct SResultRowEntryInfo pEntryInfo[]; // For each result column, there is a resultInfo -// char *key; // start key of current result row } SResultRow; typedef struct SResultRowPosition { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index e3e85bf2b8..49316ff746 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -540,6 +540,13 @@ typedef struct SFillOperatorInfo { bool multigroupResult; } SFillOperatorInfo; +typedef struct SScalarSupp { + SExprInfo* pScalarExprInfo; + int32_t numOfScalarExpr; // the number of scalar expression in group operator + SqlFunctionCtx* pScalarFuncCtx; + int32_t* rowCellInfoOffset; // offset value for each row result cell info +} SScalarSupp; + typedef struct SGroupbyOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; @@ -552,10 +559,7 @@ typedef struct SGroupbyOperatorInfo { char* keyBuf; // group by keys for hash int32_t groupKeyLen; // total group by column width SGroupResInfo groupResInfo; - SExprInfo* pScalarExprInfo; - int32_t numOfScalarExpr; // the number of scalar expression in group operator - SqlFunctionCtx* pScalarFuncCtx; - int32_t* rowCellInfoOffset; // offset value for each row result cell info + SScalarSupp scalarSup; } SGroupbyOperatorInfo; typedef struct SDataGroupInfo { @@ -578,6 +582,7 @@ typedef struct SPartitionOperatorInfo { int32_t* columnOffset; // start position for each column data void* pGroupIter; // group iterator int32_t pageIndex; // page index of current group + SScalarSupp scalarSupp; } SPartitionOperatorInfo; typedef struct SWindowRowsSup { @@ -755,6 +760,8 @@ void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); +SArray* extractPartitionColInfo(SNodeList* pNodeList); + void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); @@ -834,20 +841,17 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, const SNodeListNode* pValNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, - SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, - int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 6ba1cf859e..01ed30c189 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -28,17 +28,6 @@ typedef struct SCompSupporter { int32_t order; } SCompSupporter; -int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { - int32_t size = 0; - - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// size += pQueryAttr->pExpr1[i].base.interBytes; - } - - assert(size >= 0); - return size; -} - int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->size = 0; pResultRowInfo->cur.pageId = -1; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index de109a24ca..89692c0061 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -40,9 +40,6 @@ #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define SDATA_BLOCK_INITIALIZER \ - (SDataBlockInfo) { {0}, 0 } - #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #if 0 @@ -95,8 +92,6 @@ static void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlo static void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo); -static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); - static void releaseQueryBuf(size_t numOfTables); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); @@ -454,75 +449,6 @@ static bool chkWindowOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, - bool timeWindowInterpo) { - int64_t skey = TSKEY_INITIAL_VAL; -#if 0 - int32_t i = 0; - for (i = pResultRowInfo->size - 1; i >= 0; --i) { - SResultRow* pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { - break; - } - - // new closed result rows - if (timeWindowInterpo) { - if (pResult->endInterp && - ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { - if (i > 0) { // the first time window, the startInterp is false. - assert(pResult->startInterp); - } - - closeResultRow(pResultRowInfo, i); - } else { - skey = pResult->win.skey; - } - } else { - if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { - closeResultRow(pResultRowInfo, i); - } else { - skey = pResult->win.skey; - } - } - } - - // all result rows are closed, set the last one to be the skey - if (skey == TSKEY_INITIAL_VAL) { - if (pResultRowInfo->size == 0) { - // assert(pResultRowInfo->current == NULL); - assert(pResultRowInfo->curPos == -1); - pResultRowInfo->curPos = -1; - } else { - pResultRowInfo->curPos = pResultRowInfo->size - 1; - } - } else { - for (i = pResultRowInfo->size - 1; i >= 0; --i) { - SResultRow* pResult = pResultRowInfo->pResult[i]; - if (pResult->closed) { - break; - } - } - - if (i == pResultRowInfo->size - 1) { - pResultRowInfo->curPos = i; - } else { - pResultRowInfo->curPos = i + 1; // current not closed result object - } - } -#endif -} -// -// static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, -// bool ascQuery, bool interp) { -// if ((lastKey > pWin->ekey && ascQuery) || (lastKey < pWin->ekey && (!ascQuery))) { -// closeAllResultRows(pResultRowInfo); -// pResultRowInfo->curPos = pResultRowInfo->size - 1; -// } else { -// int32_t step = ascQuery ? 1 : -1; -// doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, interp); -// } -//} - // query_range_start, query_range_end, window_duration, window_start, window_end void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow) { pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; @@ -810,20 +736,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // _rowts/_c0, not tbname column if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) { // do nothing - } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); - pfCtx->fpSet.init(&pCtx[k], pResInfo); - - pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); - pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset - - // set the timestamp(_rowts) output buffer - if (taosArrayGetSize(pPseudoList) > 0) { - int32_t* outputColIndex = taosArrayGet(pPseudoList, 0); - pfCtx->pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput; - } - - numOfRows = pfCtx->fpSet.process(pfCtx); } else { SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); taosArrayPush(pBlockList, &pSrcBlock); @@ -886,7 +798,6 @@ int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup); assert(pResultRow != NULL); - setResultRowKey(pResultRow, pData, type); setResultRowInitCtx(pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -908,21 +819,6 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return false; } - // if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { - // // return QUERY_IS_ASC_QUERY(pQueryAttr); - // } - // - // // denote the order type - // if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { - // // return pCtx->param[0].i == pQueryAttr->order.order; - // } - - // in the reverse table scan, only the following functions need to be executed - // if (IS_REVERSE_SCAN(pRuntimeEnv) || - // (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { - // return false; - // } - return true; } @@ -3934,27 +3830,6 @@ void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows) { } } -// static STableQueryInfo* initTableQueryInfo(const STableListInfo* pTableListInfo) { -// int32_t size = taosArrayGetSize(pTableListInfo->pTableList); -// if (size == 0) { -// return NULL; -// } -// -// STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(size, sizeof(STableQueryInfo)); -// if (pTableQueryInfo == NULL) { -// return NULL; -// } -// -// for (int32_t j = 0; j < size; ++j) { -// STableKeyInfo* pk = taosArrayGet(pTableListInfo->pTableList, j); -// STableQueryInfo* pTQueryInfo = &pTableQueryInfo[j]; -// pTQueryInfo->lastKey = pk->lastKey; -// } -// -// pTableQueryInfo->lastKey = 0; -// return pTableQueryInfo; -//} - SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo) { @@ -4526,7 +4401,6 @@ static int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableU static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); -static SArray* extractPartitionColInfo(SNodeList* pNodeList); int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; @@ -4861,12 +4735,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { - SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; - SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - - SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &num); - pOptr = createPartitionOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pTaskInfo); + pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) { SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 688c576d03..23226a0134 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -320,8 +320,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order, scanFlag, true); // there is an scalar expression that needs to be calculated right before apply the group aggregation. - if (pInfo->pScalarExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->pScalarExprInfo, pBlock, pBlock, pInfo->pScalarFuncCtx, pInfo->numOfScalarExpr, NULL); + if (pInfo->scalarSup.pScalarExprInfo != NULL) { + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pScalarExprInfo, pBlock, pBlock, pInfo->scalarSup.pScalarFuncCtx, pInfo->scalarSup.numOfScalarExpr, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, pTaskInfo->code); } @@ -385,9 +385,9 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pInfo->pGroupCols = pGroupColList; pInfo->pCondition = pCondition; - pInfo->pScalarExprInfo = pScalarExprInfo; - pInfo->numOfScalarExpr = numOfScalarExpr; - pInfo->pScalarFuncCtx = createSqlFunctionCtx(pScalarExprInfo, numOfScalarExpr, &pInfo->rowCellInfoOffset); + pInfo->scalarSup.pScalarExprInfo = pScalarExprInfo; + pInfo->scalarSup.numOfScalarExpr = numOfScalarExpr; + pInfo->scalarSup.pScalarFuncCtx = createSqlFunctionCtx(pScalarExprInfo, numOfScalarExpr, &pInfo->scalarSup.rowCellInfoOffset); int32_t code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); if (code != TSDB_CODE_SUCCESS) { @@ -624,7 +624,9 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { return NULL; } - SGroupbyOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SPartitionOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { @@ -641,6 +643,14 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { break; } + // there is an scalar expression that needs to be calculated right before apply the group aggregation. + if (pInfo->scalarSupp.pScalarExprInfo != NULL) { + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSupp.pScalarExprInfo, pBlock, pBlock, pInfo->scalarSupp.pScalarFuncCtx, pInfo->scalarSupp.numOfScalarExpr, NULL); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, pTaskInfo->code); + } + } + doHashPartition(pOperator, pBlock); } @@ -665,15 +675,26 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pInfo->columnOffset); } -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pGroupCols = pGroupColList; + SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); + + pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); + + if (pPartNode->pExprs != NULL) { + pInfo->scalarSupp.numOfScalarExpr = 0; + pInfo->scalarSupp.pScalarExprInfo = createExprInfo(pPartNode->pExprs, NULL, &pInfo->scalarSupp.numOfScalarExpr); + pInfo->scalarSupp.pScalarFuncCtx = createSqlFunctionCtx( + pInfo->scalarSupp.pScalarExprInfo, pInfo->scalarSupp.numOfScalarExpr, &pInfo->scalarSupp.rowCellInfoOffset); + } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); @@ -683,16 +704,16 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* uint32_t defaultPgsz = 0; uint32_t defaultBufsz = 0; - getBufferPgSize(pResultBlock->info.rowSize, &defaultPgsz, &defaultBufsz); + getBufferPgSize(pResBlock->info.rowSize, &defaultPgsz, &defaultBufsz); int32_t code = createDiskbasedBuf(&pInfo->pBuf, defaultPgsz, defaultBufsz, pTaskInfo->id.str, TD_TMP_DIR_PATH); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->rowCapacity = blockDataGetCapacityInRow(pResultBlock, getBufPageSize(pInfo->pBuf)); - pInfo->columnOffset = setupColumnOffset(pResultBlock, pInfo->rowCapacity); - code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); + pInfo->rowCapacity = blockDataGetCapacityInRow(pResBlock, getBufPageSize(pInfo->pBuf)); + pInfo->columnOffset = setupColumnOffset(pResBlock, pInfo->rowCapacity); + code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -701,7 +722,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResultBlock; + pInfo->binfo.pRes = pResBlock; pOperator->numOfExprs = numOfCols; pOperator->pExpr = pExprInfo; pOperator->info = pInfo; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d30e4ef6db..62c2240098 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1157,6 +1157,8 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } + doFilter(pInfo->pCondition, pInfo->pRes); +#if 0 SFilterInfo* filter = NULL; int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); @@ -1202,6 +1204,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { px->info.rows = numOfRow; pInfo->pRes = px; +#endif return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -1380,6 +1383,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock); doFilterResult(pInfo); + blockDataDestroy(p); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -1469,10 +1474,10 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { p->info.rows = buildDbTableInfoBlock(p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock); - // blockDataDestroy(p); todo handle memory leak - pInfo->pRes->info.rows = p->info.rows; - return p->info.rows; + blockDataDestroy(p); + + return pInfo->pRes->info.rows; } int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4141c85366..4673123779 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -333,11 +333,6 @@ void doTimeWindowInterpolation(SIntervalAggOperatorInfo* pInfo, int32_t numOfExp continue; } - // if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { - // pCtx[k].start.key = INT64_MIN; - // continue; - // } - SFunctParam* pParam = &pCtx[k].param[0]; SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId); diff --git a/source/libs/function/inc/texpr.h b/source/libs/function/inc/texpr.h deleted file mode 100644 index 68aedcb5a2..0000000000 --- a/source/libs/function/inc/texpr.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_COMMON_EXPR_H_ -#define _TD_COMMON_EXPR_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tmsg.h" -#include "taosdef.h" -#include "tskiplist.h" -#include "function.h" - -struct tExprNode; -struct SSchema; - -#define QUERY_COND_REL_PREFIX_IN "IN|" -#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_COND_REL_PREFIX_MATCH "MATCH|" -#define QUERY_COND_REL_PREFIX_NMATCH "NMATCH|" - -#define QUERY_COND_REL_PREFIX_IN_LEN 3 -#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 -#define QUERY_COND_REL_PREFIX_MATCH_LEN 6 -#define QUERY_COND_REL_PREFIX_NMATCH_LEN 7 - -typedef bool (*__result_filter_fn_t)(const void *, void *); -typedef void (*__do_filter_suppl_fn_t)(void *, void *); - -/** - * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required - */ -typedef struct tQueryInfo { - uint8_t optr; // expression operator - SSchema sch; // schema of tags - char* q; - __compar_fn_t compare; // filter function - bool indexed; // indexed columns -} tQueryInfo; - -typedef struct SExprTraverseSupp { - __result_filter_fn_t nodeFilterFn; - __do_filter_suppl_fn_t setupInfoFn; - void *pExtInfo; -} SExprTraverseSupp; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_COMMON_EXPR_H_*/ diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index f04b4f17f9..039e0a9dfc 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -17,15 +17,7 @@ #include "os.h" #include "texception.h" -#include "taosdef.h" #include "tmsg.h" -#include "tarray.h" -#include "tbuffer.h" -#include "tcompare.h" -#include "thash.h" -#include "texpr.h" -#include "tvariant.h" -#include "tdef.h" static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); From fdee8a0cb37910013b7de71a8452ea98f5163b54 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 09:55:03 +0800 Subject: [PATCH 12/90] feat: redistribute vgroup --- source/dnode/mnode/impl/src/mndTrans.c | 19 +++---------------- source/dnode/vnode/src/vnd/vnodeSync.c | 2 -- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5b819ab092..dafb2fa1d1 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -873,7 +873,8 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) pAction->rawWritten = 0; pAction->msgSent = 0; pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT) { + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NOT_IN_NEW_CONFIG || + pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), action, pAction->epSet.inUse); @@ -1078,8 +1079,6 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (numOfActions == 0) return code; if (pTrans->redoActionPos >= numOfActions) return code; - int32_t retryTimes = 0; - for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); @@ -1128,26 +1127,14 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) pTrans->code = terrno; mError("trans:%d, %s:%d is executed and failed to sync to other mnodes since %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, terrstr()); - if (retryTimes >= 3) - break; - else - continue; } - retryTimes = 0; } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; } else { terrno = code; pTrans->code = code; - pAction->epSet.inUse++; - if (pAction->epSet.inUse >= pAction->epSet.numOfEps) pAction->epSet.inUse = 0; - mError("trans:%d, %s:%d failed to execute since %s, inUse set to %d", pTrans->id, mndTransStr(pAction->stage), - pAction->id, terrstr(), pAction->epSet.inUse); - if (retryTimes >= 3) - break; - else - continue; + break; } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index ce0c21bc35..25661d2e69 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -79,8 +79,6 @@ static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) { vError("vgId:%d, failed to transfer leader since %s", TD_VID(pVnode), terrstr()); } else { vDebug("vgId:%d, transfer leader success, propose reconfig config again", TD_VID(pVnode)); - taosMsleep(10); - code = syncPropose(pVnode->sync, &rpcMsg, false); } } From f6f4f65a56228e6276bb063a9dde2f5b2a4153d5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 16 Jun 2022 09:55:32 +0800 Subject: [PATCH 13/90] merge 3.0 --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b51ef4daf9..ce19d22ffc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4672,7 +4672,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SArray* dataReaders = taosArrayInit(8, POINTER_BYTES); createMultipleDataReaders(pTableScanNode, pHandle, pTableListInfo, dataReaders, queryId, taskId, pTagCond); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json taosArrayDestroy(groupKeys); SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, dataReaders, pHandle, pTaskInfo); From a2d1987208066569fc7d9084b804c0f547f8301c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 16 Jun 2022 10:24:01 +0800 Subject: [PATCH 14/90] show apps --- source/client/src/clientHb.c | 3 +++ source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 32 ++++++++++++++---------- source/libs/nodes/src/nodesCodeFuncs.c | 2 -- source/libs/nodes/src/nodesUtilFuncs.c | 2 -- source/libs/parser/src/parAstParser.c | 19 ++++++++++++++ 6 files changed, 42 insertions(+), 18 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 72b7988f25..ac9bca7295 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -623,6 +623,8 @@ int32_t hbGatherAppInfo(void) { req.appId = clientHbMgr.appId; taosGetAppName(req.name, NULL); } + + taosHashClear(clientHbMgr.appSummary); for (int32_t i = 0; i < sz; ++i) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -630,6 +632,7 @@ int32_t hbGatherAppInfo(void) { SAppHbReq* pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL == pApp) { memcpy(&req.summary, &pAppHbMgr->pAppInstInfo->summary, sizeof(req.summary)); + req.startTime = pAppHbMgr->startTime; taosHashPut(clientHbMgr.appSummary, &clusterId, sizeof(clusterId), &req, sizeof(req)); } else { if (pAppHbMgr->startTime < pApp->startTime) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 21161b8444..440c036d69 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -317,7 +317,7 @@ static const SSysDbTableSchema querySchema[] = { static const SSysDbTableSchema appSchema[] = { {.name = "app_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "start_time", .bytes = 8 , .type = TSDB_DATA_TYPE_TIMESTAMP}, diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 73934f5144..b0c6e57687 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -863,47 +863,53 @@ static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pApp->appId, false); + char ip[TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&ip[VARSTR_HEADER_SIZE], "%s", taosIpStr(pApp->ip)); + varDataLen(ip) = strlen(&ip[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pApp->ip, false); + colDataAppend(pColInfo, numOfRows, (const char *)ip, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pApp->pid, false); + char name[TSDB_APP_NAME_LEN + 6 + VARSTR_HEADER_SIZE] = {0}; + sprintf(&name[VARSTR_HEADER_SIZE], "%s", pApp->name); + varDataLen(name) = strlen(&name[VARSTR_HEADER_SIZE]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->name, false); + colDataAppend(pColInfo, numOfRows, (const char *)name, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->startTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->startTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.numOfInsertsReq, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertsReq, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.numOfInsertRows, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfInsertRows, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.insertElapsedTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.insertElapsedTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.insertBytes, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.insertBytes, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.fetchBytes, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.fetchBytes, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.queryElapsedTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.queryElapsedTime, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.numOfSlowQueries, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.numOfSlowQueries, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.totalRequests, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.totalRequests, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->summary.currentRequests, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->summary.currentRequests, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)pApp->lastAccessTimeMs, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pApp->lastAccessTimeMs, false); numOfRows++; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index be1929d554..59dfb31600 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -172,8 +172,6 @@ const char* nodesNodeName(ENodeType type) { return "ShowSubscribesStmt"; case QUERY_NODE_SHOW_SMAS_STMT: return "ShowSmasStmt"; - case QUERY_NODE_SHOW_CONFIGS_STMT: - return "ShowConfigsStmt"; case QUERY_NODE_SHOW_QUERIES_STMT: return "ShowQueriesStmt"; case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 0f1dbfc13a..c9a188566d 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -198,7 +198,6 @@ SNode* nodesMakeNode(ENodeType type) { case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_SUBSCRIBES_STMT: case QUERY_NODE_SHOW_SMAS_STMT: - case QUERY_NODE_SHOW_CONFIGS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_VNODES_STMT: @@ -610,7 +609,6 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_SUBSCRIBES_STMT: case QUERY_NODE_SHOW_SMAS_STMT: - case QUERY_NODE_SHOW_CONFIGS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 1aa1320a3c..1e968fe1fa 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -387,6 +387,21 @@ static int32_t collectMetaKeyFromShowQueries(SCollectMetaKeyCxt* pCxt, SShowStmt pCxt->pMetaCache); } +static int32_t collectMetaKeyFromShowConfigs(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CONFIGS, + pCxt->pMetaCache); +} + +static int32_t collectMetaKeyFromShowVariables(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CONFIGS, + pCxt->pMetaCache); +} + +static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_APPS, + pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromShowTransactions(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_TRANS, pCxt->pMetaCache); @@ -461,6 +476,10 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowConnections(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_QUERIES_STMT: return collectMetaKeyFromShowQueries(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_VARIABLE_STMT: + return collectMetaKeyFromShowVariables(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_APPS_STMT: + return collectMetaKeyFromShowApps(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return collectMetaKeyFromShowTransactions(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_DELETE_STMT: From 803249837a00f12207a455fbc228209cf2372360 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 10:29:30 +0800 Subject: [PATCH 15/90] feat(stream): drop stream --- include/common/tmsg.h | 6 ++--- source/dnode/mnode/impl/src/mndStream.c | 31 ++++++++++++++----------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4894bef8ff..7c0e8206e1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2008,7 +2008,7 @@ typedef struct { char sql[TSDB_SHOW_SQL_LEN]; uint64_t queryId; int64_t useconds; - int64_t stime; // timestamp precision ms + int64_t stime; // timestamp precision ms int64_t reqRid; int32_t pid; bool stableQuery; @@ -2257,8 +2257,8 @@ typedef struct { } SMqVDeleteRsp; typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - int64_t streamId; + char name[TSDB_STREAM_FNAME_LEN]; + int8_t igNotExists; } SMDropStreamTaskReq; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 24e41435e9..491835fc15 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -278,8 +278,8 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast } static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { - SStbObj *pStb = NULL; - SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SDbObj *pDb = NULL; SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); @@ -321,7 +321,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -520,13 +519,19 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { /*SDbObj *pDb = NULL;*/ /*SUserObj *pUser = NULL;*/ - SMDropStreamTaskReq dropStreamReq = *(SMDropStreamTaskReq *)pReq->pCont; + SMDropStreamTaskReq dropReq = *(SMDropStreamTaskReq *)pReq->pCont; - pStream = mndAcquireStream(pMnode, dropStreamReq.name); + pStream = mndAcquireStream(pMnode, dropReq.name); if (pStream == NULL) { - terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; - return -1; + if (dropReq.igNotExists) { + mDebug("stream:%s, not exist, ignore not exist is set", dropReq.name); + code = 0; + goto DROP_STREAM_OVER; + } else { + terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + return -1; + } } #if 0 @@ -539,19 +544,19 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { - mError("stream:%s, failed to drop since %s", dropStreamReq.name, terrstr()); - return -1; + mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); + return code; } - mDebug("trans:%d, used to drop stream:%s", pTrans->id, dropStreamReq.name); + mDebug("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); // drop all tasks if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { - mError("stream:%s, failed to drop task since %s", dropStreamReq.name, terrstr()); - return -1; + mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); + return code; } DROP_STREAM_OVER: - return 0; + return code; } static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { From 63884b554bf9104ffc6bb7e13d0efd5483b3b1be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 10:51:29 +0800 Subject: [PATCH 16/90] fix(query): set correct sort buffer page size. --- source/common/src/tdatablock.c | 1 + source/libs/executor/inc/tsort.h | 7 +++++++ source/libs/executor/src/sortoperator.c | 27 +++++++++++++++---------- source/libs/executor/src/tsort.c | 26 +++++++++++++++--------- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5a2aaed74e..6a4ddacf60 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1226,6 +1226,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfo.info = p->info; + colInfo.hasNull = true; taosArrayPush(pBlock->pDataBlock, &colInfo); } diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index fd3581e2bf..e731c55a7d 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -151,6 +151,13 @@ SSDataBlock* tsortGetSortedDataBlock(const SSortHandle* pSortHandle); */ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle); +/** + * get proper sort buffer pages according to the row size + * @param rowSize + * @return + */ +int32_t getProperSortPageSize(size_t rowSize); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 1008a5263c..98a1aacbb6 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -420,24 +420,29 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, goto _error; } - pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = pSortInfo; + pInfo->binfo.pRes = pResBlock; + pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; - pInfo->pInputBlock = pInputBlock; - pOperator->name = "MultiwaySortMerge"; + pInfo->pInputBlock = pInputBlock; + pOperator->name = "MultiwaySortMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - - pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; - pInfo->sortBufSize = pInfo->bufPageSize * 16; - pInfo->hasGroupId = false; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pInfo->hasGroupId = false; pInfo->prefetchedTuple = NULL; pOperator->pTaskInfo = pTaskInfo; + + pInfo->bufPageSize = getProperSortPageSize(rowSize); + + uint32_t numOfSources = taosArrayGetSize(pSortInfo); + numOfSources = MAX(2, numOfSources); + + pInfo->sortBufSize = numOfSources * pInfo->bufPageSize; + pOperator->fpSet = createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 1473bd81bb..daaba74d7c 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -532,6 +532,19 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return 0; } +int32_t getProperSortPageSize(size_t rowSize) { + uint32_t defaultPageSize = 4096; + + uint32_t pgSize = 0; + if (rowSize * 4 > defaultPageSize) { + pgSize = rowSize * 4; + } else { + pgSize = defaultPageSize; + } + + return pgSize; +} + static int32_t createInitialSources(SSortHandle* pHandle) { size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; @@ -557,14 +570,9 @@ static int32_t createInitialSources(SSortHandle* pHandle) { if (!hasGroupId) { // calculate the buffer pages according to the total available buffers. - int32_t rowSize = blockDataGetRowSize(pBlock); - if (rowSize * 4 > 4096) { - pHandle->pageSize = rowSize * 4; - } else { - pHandle->pageSize = 4096; - } - - // todo!! + pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock)); + + // todo, number of pages are set according to the total available sort buffer pHandle->numOfPages = 1024; sortBufSize = pHandle->numOfPages * pHandle->pageSize; @@ -577,7 +585,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { if (pHandle->beforeFp != NULL) { pHandle->beforeFp(pBlock, pHandle->param); } - // todo relocate the columns + int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); if (code != 0) { return code; From f89fcece482499638cf6d4623111ca1902f164e7 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 16 Jun 2022 10:46:20 +0800 Subject: [PATCH 17/90] Update index.mdx --- docs/en/07-develop/02-model/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/07-develop/02-model/index.mdx b/docs/en/07-develop/02-model/index.mdx index 86853aaaa3..e0378cc77c 100644 --- a/docs/en/07-develop/02-model/index.mdx +++ b/docs/en/07-develop/02-model/index.mdx @@ -37,7 +37,7 @@ USE power; ## Create STable -In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](/tdinternal/arch#model_table1), the SQL statement below can be used to create the super table. +In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](/concept/#model_table1), the SQL statement below can be used to create the super table. ```sql CREATE STable meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int); From 58051169d2102ad1b9c43f62673d40e34d7aab63 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 10:54:13 +0800 Subject: [PATCH 18/90] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntriesReply.c | 8 +-- source/libs/sync/src/syncSnapshot.c | 70 ++++++++++--------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 0e116e13ee..fbe3319675 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -191,17 +191,17 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries char* s = snapshotSender2Str(pSender); sDebug( "vgId:%d sync event snapshot send to %s:%d start sender first time, lastApplyIndex:%ld lastApplyTerm:%lu " - "lastConfigIndex:%ld" + "lastConfigIndex:%ld privateTerm:%lu " "sender:%s", ths->vgId, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - pSender->snapshot.lastConfigIndex, s); + pSender->snapshot.lastConfigIndex, pSender->privateTerm, s); taosMemoryFree(s); } else { sDebug( "vgId:%d sync event snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " - "lastApplyTerm:%lu lastConfigIndex:%ld", + "lastApplyTerm:%lu lastConfigIndex:%ld privateTerm:%lu", ths->vgId, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - pSender->snapshot.lastConfigIndex); + pSender->snapshot.lastConfigIndex, pSender->privateTerm); } } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index afebfa798e..c92920a936 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -142,17 +142,17 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( "vgId:%d sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " - "lastConfigIndex:%ld send " + "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, msgStr); + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( "vgId:%d sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " - "lastConfigIndex:%ld", + "lastConfigIndex:%ld privateTerm:%lu", pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -280,24 +280,24 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( "vgId:%d sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " - "lastConfigIndex:%ld send " + "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, msgStr); + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( "vgId:%d sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " - "lastConfigIndex:%ld", + "lastConfigIndex:%ld privateTerm:%lu", pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } } else { sDebug( "vgId:%d sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " - "lastConfigIndex:%ld", + "lastConfigIndex:%ld privateTerm:%lu", pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -328,12 +328,12 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); - sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d send msg:%s", pSender->pSyncNode->vgId, - host, port, pSender->seq, pSender->ack, msgStr); + sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu send msg:%s", + pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { - sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d", pSender->pSyncNode->vgId, host, port, - pSender->seq, pSender->ack); + sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu", pSender->pSyncNode->vgId, + host, port, pSender->seq, pSender->ack, pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -567,15 +567,16 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( "vgId:%d sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld, recv msg:%s", + "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, - msgStr); + pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( "vgId:%d sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld", - pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + "lastConfigIndex:%ld privateTerm:%lu", + pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, + pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -638,17 +639,17 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { sDebug( "vgId:%d sync event snapshot recv from %s:%d finish, update log begin index:%ld, " "snapshot.lastApplyIndex:%ld, " - "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, raft log:%s", + "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu, raft log:%s", pSyncNode->vgId, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - snapshot.lastConfigIndex, logSimpleStr); + snapshot.lastConfigIndex, pReceiver->privateTerm, logSimpleStr); taosMemoryFree(logSimpleStr); } else { sDebug( "vgId:%d sync event snapshot recv from %s:%d finish, update log begin index:%ld, " "snapshot.lastApplyIndex:%ld, " - "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld", + "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu", pSyncNode->vgId, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - snapshot.lastConfigIndex); + snapshot.lastConfigIndex, pReceiver->privateTerm); } pReceiver->pWriter = NULL; @@ -660,16 +661,16 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( "vgId:%d sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld, recv msg:%s", + "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex, msgStr); + pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( "vgId:%d sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld", + "lastConfigIndex:%ld, privateTerm:%lu", pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex); + pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -685,17 +686,17 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( "vgId:%d sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld, recv " + "lastConfigIndex:%ld, privateTerm:%lu, recv " "msg:%s", pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex, msgStr); + pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( "vgId:%d sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld", + "lastConfigIndex:%ld, privateTerm:%lu", pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex); + pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { @@ -716,15 +717,16 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( "vgId:%d sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld, recv msg:%s", + "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, - msgStr); + pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( "vgId:%d sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, " - "lastConfigIndex:%ld", - pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + "lastConfigIndex:%ld, privateTerm:%lu", + pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, + pReceiver->privateTerm); } } else { From 61e3250ee76560cef9d24f871453e34116b28a7c Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 16 Jun 2022 11:17:36 +0800 Subject: [PATCH 19/90] docs: correct TAOS CLI to TDengine CLI --- docs/en/05-get-started/index.md | 4 ++-- docs/en/07-develop/03-insert-data/01-sql-writing.mdx | 2 +- docs/zh/05-get-started/index.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index 56958ef3ec..9c5a853820 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -1,6 +1,6 @@ --- title: Get Started -description: 'Install TDengine from Docker image, apt-get or package, and run TAOS CLI and taosBenchmark to experience the features' +description: 'Install TDengine from Docker image, apt-get or package, and run TDengine CLI and taosBenchmark to experience the features' --- import Tabs from "@theme/Tabs"; @@ -120,7 +120,7 @@ select * from t; Query OK, 2 row(s) in set (0.003128s) ``` -Besides executing SQL commands, system administrators can check running status, add/drop user accounts and manage the running instances. TAOS CLI with client driver can be installed and run on either Linux or Windows machines. For more details on CLI, please [check here](../reference/taos-shell/). +Besides executing SQL commands, system administrators can check running status, add/drop user accounts and manage the running instances. TDengine CLI with client driver can be installed and run on either Linux or Windows machines. For more details on CLI, please [check here](../reference/taos-shell/). ## Experience the blazing fast speed diff --git a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx index 397b1a14fd..d8c4453f40 100644 --- a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx @@ -22,7 +22,7 @@ import CStmt from "./_c_stmt.mdx"; ## Introduction -Application programs can execute `INSERT` statement through connectors to insert rows. The TAOS CLI can also be used to manually insert data. +Application programs can execute `INSERT` statement through connectors to insert rows. The TDengine CLI can also be used to manually insert data. ### Insert Single Row diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 878d7f0202..818027174e 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -1,6 +1,6 @@ --- title: 立即开始 -description: '从 Docker，安装包或使用 apt-get 快速安装 TDengine, 通过命令行程序TAOS CLI和工具 taosdemo 快速体验 TDengine 功能' +description: '从 Docker，安装包或使用 apt-get 快速安装 TDengine, 通过命令行程序TDengine CLI和工具 taosdemo 快速体验 TDengine 功能' --- import Tabs from "@theme/Tabs"; @@ -122,7 +122,7 @@ select * from t; Query OK, 2 row(s) in set (0.003128s) ``` -除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TAOS CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../reference/taos-shell/) +除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../reference/taos-shell/) ## 使用 taosBenchmark 体验写入速度 From 2360f072a84dba8086dda533e8297d2821b5b8b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 11:43:11 +0800 Subject: [PATCH 20/90] fix(query): add indef process procedure. --- include/libs/executor/executor.h | 9 +-------- source/libs/executor/inc/executorimpl.h | 4 ++-- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/executorMain.c | 4 ++-- source/libs/executor/src/executorimpl.c | 17 ++++++++++++++++- source/libs/qworker/inc/qwMsg.h | 2 +- source/libs/qworker/src/qwMsg.c | 4 +--- source/libs/qworker/src/qworker.c | 6 +++--- 8 files changed, 27 insertions(+), 21 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 3f05c27453..fdedb947d7 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -95,7 +95,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo * @return */ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model); + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model); /** * @@ -159,13 +159,6 @@ int64_t qGetQueriedTableUid(qTaskInfo_t tinfo); */ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); -/** - * Update the table id list of a given query. - * @param uid child table uid - * @param type operation type: ADD|DROP - * @return - */ -int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type); void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 758c031028..b435446513 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -189,7 +189,7 @@ typedef struct SExecTaskInfo { } schemaVer; STableListInfo tableqinfoList; // this is a table list - char* sql; // query sql string + const char* sql; // query sql string jmp_buf env; // jump to this position when error happens. EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] struct SOperatorInfo* pRoot; @@ -894,7 +894,7 @@ int32_t decodeOperator(SOperatorInfo* ops, char* data, int32_t length); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - EOPTR_EXEC_MODEL model); + const char* sql, EOPTR_EXEC_MODEL model); int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo** pRes, int32_t* capacity, int32_t* resNum); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index bebfc75f17..b1d076e8f5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -121,7 +121,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { // TODO: destroy SSubplan & pTaskInfo terrno = code; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index b1125c8e80..00158d7024 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -31,13 +31,13 @@ static void initRefPool() { } int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model) { + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; taosThreadOnce(&initPoolOnce, initRefPool); - int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, model); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d066284c56..9ce22431e1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -736,6 +736,20 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // _rowts/_c0, not tbname column if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) { // do nothing + } else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); + pfCtx->fpSet.init(&pCtx[k], pResInfo); + + pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); + pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset + + // set the timestamp(_rowts) output buffer + if (taosArrayGetSize(pPseudoList) > 0) { + int32_t* outputColIndex = taosArrayGet(pPseudoList, 0); + pfCtx->pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput; + } + + numOfRows = pfCtx->fpSet.process(pfCtx); } else { SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); taosArrayPush(pBlockList, &pSrcBlock); @@ -5183,7 +5197,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT } int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - EOPTR_EXEC_MODEL model) { + const char* sql, EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -5193,6 +5207,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } + (*pTaskInfo)->sql = sql; (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond); if (NULL == (*pTaskInfo)->pRoot) { diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index 29861d87ac..ecff861f50 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -24,7 +24,7 @@ extern "C" { #include "dataSinkMgt.h" int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain); +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain, const char* sql); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 848a0420ca..5635ec8fc6 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -308,10 +308,8 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, sql); - taosMemoryFreeClear(sql); - - QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType, msg->explain)); + QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType, msg->explain, sql)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 451607e7d0..57ebb89ed2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -510,7 +510,7 @@ _return: } -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain, const char* sql) { int32_t code = 0; bool queryRsped = false; SSubplan *plan = NULL; @@ -537,7 +537,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex ctx->plan = plan; - code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, sql, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -938,7 +938,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SRpcMsg *pRsp, SDeleteRes ctx.plan = plan; - code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, NULL, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); From 102fdf2916aadfe0f438806fd7c188d3580a4537 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 11:52:58 +0800 Subject: [PATCH 21/90] fix(query): fix the syntax error in unit test. --- source/libs/executor/test/executorTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 880ac6c78e..dba2fb897a 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -945,7 +945,7 @@ TEST(testCase, build_executor_tree_Test) { int32_t code = qStringToSubplan(msg, &plan); ASSERT_EQ(code, 0); - code = qCreateExecTask(&handle, 2, 1, plan, (void**)&pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(&handle, 2, 1, plan, (void**)&pTaskInfo, &sinkHandle, NULL, OPTR_EXEC_MODEL_BATCH); ASSERT_EQ(code, 0); } #if 0 From ed532ed7acb5e0e4916b09e2e1e9e60308790380 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 12:01:13 +0800 Subject: [PATCH 22/90] refactor: adjust logs --- source/dnode/mnode/impl/src/mndDnode.c | 1 + source/dnode/mnode/impl/src/mndMain.c | 1 + source/dnode/mnode/impl/src/mndMnode.c | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 53cbb9b669..6d37143fc9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -370,6 +370,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnodeObj *pObj = mndAcquireMnode(pMnode, pDnode->id); if (pObj != NULL) { if (pObj->state != statusReq.mload.syncState) { + mInfo("dnode:%d, mnode syncstate from %s to %s", pObj->id, syncStr(pObj->state), syncStr(statusReq.mload.syncState)); pObj->state = statusReq.mload.syncState; pObj->stateStartTime = taosGetTimestampMs(); } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 00118eac1e..e141e7dfb3 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -699,6 +699,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { pLoad->syncState = syncGetMyRole(pMnode->syncMgmt.sync); + mTrace("mnode current syncstate is %s", syncStr(pLoad->syncState)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 7ec490d52f..e82c479cdd 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -667,6 +667,10 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB } if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) { roles = syncStr(pObj->state); + if (pObj->state == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) { + roles = syncStr(TAOS_SYNC_STATE_ERROR); + mError("mnode:%d, is leader too", pObj->id); + } } char b2[12 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->pMeta->pSchemas[cols].bytes); From 75e0eaa53fed96fa0a766475dcb57ebdf1c7032c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 13:53:02 +0800 Subject: [PATCH 23/90] fix(query): add null pointer check. --- source/libs/executor/src/tsort.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index daaba74d7c..5bcd58f8db 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -204,7 +204,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int if (pHandle->type == SORT_SINGLESOURCE_SORT) { for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + if (taosArrayGetSize(pSource->pageIdList) == 0) { + return TSDB_CODE_SUCCESS; + } + + SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); code = blockDataFromBuf(pSource->src.pBlock, pPage); From 0671daa70b4db0b689df34543edd588e5a2249ed Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 16 Jun 2022 14:07:15 +0800 Subject: [PATCH 24/90] chore: update taos-tools (#13889) for 3.0 --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index 3d5aa76f8c..0a81480420 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 3d5aa76f8c718dcffa100b45e4cbf313d499c356 +Subproject commit 0a81480420d6601bbdb57770ee64e40f24c4ea83 From 2ba3b7538c50b0d7267d0c056b962e0d7f90aebe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 14:09:14 +0800 Subject: [PATCH 25/90] test: redistribute vgroup --- ...redistribute_vgroup_replica3_v1_leader.sim | 166 +++++------------- 1 file changed, 48 insertions(+), 118 deletions(-) diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim index 90f7b1ad1f..7b52b51306 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim @@ -4,6 +4,11 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1 system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start @@ -128,9 +133,9 @@ if $rows != 1 then endi if $data(2)[4] == leader then $leaderExist = 1 - $leaderVnode = 4 - $follower1 = 2 - $follower2 = 3 + $leaderVnode = 2 + $follower1 = 3 + $follower2 = 4 endi if $data(2)[6] == leader then $leaderExist = 1 @@ -140,9 +145,9 @@ if $data(2)[6] == leader then endi if $data(2)[8] == leader then $leaderExist = 1 - $leaderVnode = 2 - $follower1 = 3 - $follower2 = 4 + $leaderVnode = 4 + $follower1 = 2 + $follower2 = 3 endi if $leaderExist != 1 then goto step3 @@ -160,110 +165,22 @@ if $rows != 1 then return -1 endi -<<<<<<< HEAD -print =============== step32: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -<<<<<<< HEAD:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -======= -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 - ->>>>>>> origin/3.0:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -======= print =============== step32: move leader print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 sql show d1.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 ->>>>>>> origin/3.0 sql show d1.tables if $rows != 1 then return -1 endi -<<<<<<< HEAD +return + print =============== step33: move follower1 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 -<<<<<<< HEAD:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -======= -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 - ->>>>>>> origin/3.0:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -======= -print =============== step33: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 - ->>>>>>> origin/3.0 -sql show d1.tables -if $rows != 1 then - return -1 -endi - -<<<<<<< HEAD -print =============== step34: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 -sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 -<<<<<<< HEAD:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -======= -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 - ->>>>>>> origin/3.0:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -======= -print =============== step34: move follower1 -print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode -sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 - ->>>>>>> origin/3.0 -sql show d1.tables -if $rows != 1 then - return -1 -endi - -<<<<<<< HEAD -print =============== step35: move follower1 -print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 -sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 -<<<<<<< HEAD:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -sql show d1.tables -if $rows != 1 then - return -1 -endi -======= -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 ->>>>>>> origin/3.0:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -======= -print =============== step35: move 5 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode $follower2 -sql show d1.vgroups -print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 ->>>>>>> origin/3.0 - -sql show d1.tables -if $rows != 1 then - return -1 -endi - -<<<<<<< HEAD -print =============== step36: move follower2 print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 -======= -print =============== step36: move follower1 -print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 -sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 ->>>>>>> origin/3.0 sql show d1.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -272,11 +189,40 @@ if $rows != 1 then return -1 endi -<<<<<<< HEAD -print =============== step37: move follower1 -======= +print =============== step34: move follower2 +print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $leaderVnode +sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $leaderVnode +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step35: move leader +print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $follower1 +sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $follower1 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + +print =============== step36: move follower1 +print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode 5 +sql show d1.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 + +sql show d1.tables +if $rows != 1 then + return -1 +endi + print =============== step37: move follower2 ->>>>>>> origin/3.0 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql show d1.vgroups @@ -287,17 +233,9 @@ if $rows != 1 then return -1 endi -<<<<<<< HEAD -<<<<<<< HEAD:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -======= -print =============== step38: move follower2 -print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 -sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 -======= print =============== step38: move leader print redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode $follower2 ->>>>>>> origin/3.0 sql show d1.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -305,18 +243,10 @@ sql show d1.tables if $rows != 1 then return -1 endi -<<<<<<< HEAD ->>>>>>> origin/3.0:tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim print =============== step39: move follower1 -print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 -sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower1 -======= - -print =============== step39: move 5 -print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 -sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1 ->>>>>>> origin/3.0 +print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 +sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode $follower2 sql show d1.vgroups print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 From 77b03ac821c8c03d8f583d2e52203dd7021e605a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 14:40:42 +0800 Subject: [PATCH 26/90] refactor(sync): add restore finish when become leader again --- include/libs/sync/sync.h | 2 +- source/dnode/mnode/impl/src/mndSync.c | 12 +-- source/libs/sync/inc/syncInt.h | 3 + source/libs/sync/src/syncMain.c | 129 +++++++++++++++++++------ source/libs/sync/src/syncRaftLog.c | 7 +- source/libs/sync/src/syncRespMgr.c | 15 +-- source/libs/sync/src/syncSnapshot.c | 7 ++ source/libs/sync/test/syncTestTool.cpp | 8 +- 8 files changed, 134 insertions(+), 49 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 2d21b3531a..e694203563 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -24,7 +24,7 @@ extern "C" { #include "tdef.h" #include "tmsgcb.h" -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 typedef uint64_t SyncNodeId; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 63708aef8a..3a243e36d3 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -221,17 +221,17 @@ void mndCleanupSync(SMnode *pMnode) { int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; - SRpcMsg rsp = {.code = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; - rsp.pCont = rpcMallocCont(rsp.contLen); - if (rsp.pCont == NULL) return -1; - memcpy(rsp.pCont, pRaw, rsp.contLen); + SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; + req.pCont = rpcMallocCont(req.contLen); + if (req.pCont == NULL) return -1; + memcpy(req.pCont, pRaw, req.contLen); pMgmt->errCode = 0; pMgmt->transId = transId; mTrace("trans:%d, will be proposed", pMgmt->transId); const bool isWeak = false; - int32_t code = syncPropose(pMgmt->sync, &rsp, isWeak); + int32_t code = syncPropose(pMgmt->sync, &req, isWeak); if (code == 0) { tsem_wait(&pMgmt->syncSem); } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { @@ -242,7 +242,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { terrno = TSDB_CODE_APP_ERROR; } - rpcFreeCont(rsp.pCont); + rpcFreeCont(req.pCont); if (code != 0) { mError("trans:%d, failed to propose, code:0x%x", pMgmt->transId, code); return code; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 8a44dcd9bc..b00c7cbda1 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -195,6 +195,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); +bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config); void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex, bool* isDrop); SSyncNode* syncNodeAcquire(int64_t rid); @@ -230,6 +231,8 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncInd int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag); +int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg); + bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a0d8487b63..5e3c1adf32 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -185,7 +185,9 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg ASSERT(rid == pSyncNode->rid); int32_t ret = 0; - bool IamInNew = false; + bool IamInNew = syncNodeInConfig(pSyncNode, pNewCfg); + +#if 0 for (int i = 0; i < pNewCfg->replicaNum; ++i) { if (strcmp((pNewCfg->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && (pNewCfg->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { @@ -201,6 +203,7 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg } */ } +#endif if (!IamInNew) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -228,7 +231,9 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { } ASSERT(rid == pSyncNode->rid); - bool IamInNew = false; + bool IamInNew = syncNodeInConfig(pSyncNode, pNewCfg); + +#if 0 for (int i = 0; i < pNewCfg->replicaNum; ++i) { if (strcmp((pNewCfg->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && (pNewCfg->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { @@ -247,6 +252,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { } */ } +#endif if (!IamInNew) { sError("sync reconfig error, not in new config"); @@ -556,7 +562,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return -1; } assert(rid == pSyncNode->rid); - sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); + sDebug("vgId:%d sync event propose msgType:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), pMsg->msgType); ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -565,7 +571,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; - sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); + sDebug("vgId:%d sync event propose msgType:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), pMsg->msgType); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -1256,9 +1262,36 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { return s; } -void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex, bool* isDrop) { +bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { + bool b1 = false; + bool b2 = false; + + for (int i = 0; i < config->replicaNum; ++i) { + if (strcmp((config->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && + (config->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { + b1 = true; + break; + } + } + + for (int i = 0; i < config->replicaNum; ++i) { + SRaftId raftId; + raftId.addr = syncUtilAddr2U64((config->nodeInfo)[i].nodeFqdn, (config->nodeInfo)[i].nodePort); + raftId.vgId = pSyncNode->vgId; + + if (syncUtilSameId(&raftId, &(pSyncNode->myRaftId))) { + b2 = true; + break; + } + } + + ASSERT(b1 == b2); + return b1; +} + +void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex, bool* isDrop) { SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg; - pSyncNode->pRaftCfg->cfg = *newConfig; + pSyncNode->pRaftCfg->cfg = *pNewConfig; pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex; int32_t ret = 0; @@ -1270,7 +1303,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { oldSenders[i] = (pSyncNode->senders)[i]; - sDebug("vgId:%d sync event save senders %d, %p", pSyncNode->vgId, i, oldSenders[i]); + sDebug("vgId:%d sync event save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, i, oldSenders[i], + oldSenders[i]->privateTerm); if (gRaftDetailLog) { ; } @@ -1322,8 +1356,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event reset sender for %lu, newIndex:%d, %s:%d, %p", pSyncNode->vgId, - (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); + sDebug("vgId:%d sync event reset sender for %lu, newIndex:%d, %s:%d, %p, privateTerm:%lu", pSyncNode->vgId, + (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j], oldSenders[j]->privateTerm); (pSyncNode->senders)[i] = oldSenders[j]; oldSenders[j] = NULL; reset = true; @@ -1341,7 +1375,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sDebug("vgId:%d sync event create new sender %p replicaIndex:%d", pSyncNode->vgId, (pSyncNode->senders)[i], i); + sDebug("vgId:%d sync event create new sender %p replicaIndex:%d, privateTerm:%lu", pSyncNode->vgId, + (pSyncNode->senders)[i], i, (pSyncNode->senders)[i]->privateTerm); } } @@ -1354,8 +1389,10 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l } } - bool IamInOld = false; - bool IamInNew = false; + bool IamInOld = syncNodeInConfig(pSyncNode, &oldConfig); + bool IamInNew = syncNodeInConfig(pSyncNode, pNewConfig); + +#if 0 for (int i = 0; i < oldConfig.replicaNum; ++i) { if (strcmp((oldConfig.nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && (oldConfig.nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { @@ -1371,6 +1408,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l break; } } +#endif *isDrop = true; if (IamInOld && !IamInNew) { @@ -1379,6 +1417,10 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l *isDrop = false; } + // may be add me to a new raft group + if (IamInOld && IamInNew && oldConfig.replicaNum == 1) { + } + if (IamInNew) { pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal } @@ -1404,14 +1446,17 @@ void syncNodeRelease(SSyncNode* pNode) { taosReleaseRef(tsNodeRefId, pNode->rid) void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, term); - syncNodeBecomeFollower(pSyncNode, "update term"); + char tmpBuf[64]; + snprintf(tmpBuf, sizeof(tmpBuf), "update term to %lu", term); + syncNodeBecomeFollower(pSyncNode, tmpBuf); raftStoreClearVote(pSyncNode->pRaftStore); } } void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become follower, isStandBy:%d, replicaNum:%d, %s", pSyncNode->vgId, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, debugStr); + sDebug("vgId:%d sync event become follower, isStandBy:%d, currentTerm:%lu, replicaNum:%d, restoreFinish:%d, %s", + pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftStore->currentTerm, pSyncNode->replicaNum, + pSyncNode->restoreFinish, debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1445,8 +1490,12 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become leader, isStandBy:%d, replicaNum:%d %s", pSyncNode->vgId, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, debugStr); + // reset restoreFinish + pSyncNode->restoreFinish = false; + + sDebug("vgId:%d sync event become leader, isStandBy:%d, currentTerm:%lu, replicaNum:%d, restoreFinish:%d, %s", + pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftStore->currentTerm, pSyncNode->replicaNum, + pSyncNode->restoreFinish, debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -2028,11 +2077,11 @@ static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE pSyncLeaderTransfer->newLeaderId.addr); */ - sDebug("vgId:%d sync event, begin leader transfer", ths->vgId); + sDebug("vgId:%d sync event begin leader transfer", ths->vgId); if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { - sDebug("vgId:%d sync event, maybe leader transfer to %s:%d %lu", ths->vgId, + sDebug("vgId:%d sync event maybe leader transfer to %s:%d %lu", ths->vgId, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); @@ -2067,6 +2116,21 @@ static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE return 0; } +int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) { + for (int i = 0; i < pNewCfg->replicaNum; ++i) { + SRaftId raftId; + raftId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + raftId.vgId = ths->vgId; + + if (syncUtilSameId(&(ths->myRaftId), &raftId)) { + pNewCfg->myIndex = i; + return 0; + } + } + + return -1; +} + static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; @@ -2075,19 +2139,23 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE ASSERT(ret == 0); // update new config myIndex - bool IamInNew = false; - for (int i = 0; i < newSyncCfg.replicaNum; ++i) { - if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && - ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { - newSyncCfg.myIndex = i; - IamInNew = true; - break; - } - } + syncNodeUpdateNewConfigIndex(ths, &newSyncCfg); + + bool IamInNew = syncNodeInConfig(ths, &newSyncCfg); + + /* + for (int i = 0; i < newSyncCfg.replicaNum; ++i) { + if (strcmp(ths->myNodeInfo.nodeFqdn, (newSyncCfg.nodeInfo)[i].nodeFqdn) == 0 && + ths->myNodeInfo.nodePort == (newSyncCfg.nodeInfo)[i].nodePort) { + newSyncCfg.myIndex = i; + IamInNew = true; + break; + } + } + */ bool isDrop; - // if (IamInNew || (!IamInNew && ths->state != TAOS_SYNC_STATE_LEADER)) { if (IamInNew) { syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); @@ -2186,7 +2254,8 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sDebug("vgId:%d sync event restore finish, index:%ld", ths->vgId, pEntry->index); + sDebug("vgId:%d sync event restore finish, %s, index:%ld", ths->vgId, syncUtilState2String(ths->state), + pEntry->index); } } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 996cd12e4a..79ce84abe9 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -162,9 +162,10 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); - sDebug("vgId:%d sync event write index:%ld, %s, isStandBy:%d, msgType:%s, originalRpcType:%s", pData->pSyncNode->vgId, - pEntry->index, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftCfg->isStandBy, - TMSG_INFO(pEntry->msgType), TMSG_INFO(pEntry->originalRpcType)); + sDebug("vgId:%d sync event write index:%ld, %s, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", + pData->pSyncNode->vgId, pEntry->index, syncUtilState2String(pData->pSyncNode->state), + pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), pEntry->msgType, + TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); return code; } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 5087cacd02..adc67b5d8d 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -45,8 +45,9 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event resp mgr add, type:%s seq:%lu handle:%p", pSyncNode->vgId, - TMSG_INFO(pStub->rpcMsg.msgType), keyCode, pStub->rpcMsg.info.handle); + sDebug("vgId:%d sync event resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, + TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, + pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return keyCode; @@ -69,8 +70,9 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event resp mgr get, type:%s seq:%lu handle:%p", pSyncNode->vgId, - TMSG_INFO(pStub->rpcMsg.msgType), index, pStub->rpcMsg.info.handle); + sDebug("vgId:%d sync event resp mgr get, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, + TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, + pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return 1; // get one object @@ -87,8 +89,9 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event resp mgr get and del, type:%s seq:%lu handle:%p", pSyncNode->vgId, - TMSG_INFO(pStub->rpcMsg.msgType), index, pStub->rpcMsg.info.handle); + sDebug("vgId:%d sync event resp mgr get and del, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, + TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, + pStub->rpcMsg.info.ahandle); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); taosThreadMutexUnlock(&(pObj->mutex)); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index c92920a936..545ea953f8 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -591,6 +591,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { int32_t oldReplicaNum = pSyncNode->replicaNum; + // update new config myIndex + SSyncCfg newSyncCfg = pMsg->lastConfig; + syncNodeUpdateNewConfigIndex(pSyncNode, &newSyncCfg); + bool IamInNew = syncNodeInConfig(pSyncNode, &newSyncCfg); + +#if 0 // update new config myIndex bool IamInNew = false; SSyncCfg newSyncCfg = pMsg->lastConfig; @@ -602,6 +608,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { break; } } +#endif bool isDrop; if (IamInNew) { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 21049454f4..1eac372bd4 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -297,7 +297,7 @@ void usage(char* exe) { SRpcMsg* createRpcMsg(int i, int count, int myIndex) { SRpcMsg* pMsg = (SRpcMsg*)taosMemoryMalloc(sizeof(SRpcMsg)); memset(pMsg, 0, sizeof(SRpcMsg)); - pMsg->msgType = 9999; + pMsg->msgType = TDMT_VND_SUBMIT; pMsg->contLen = 256; pMsg->pCont = rpcMallocCont(pMsg->contLen); snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); @@ -384,8 +384,10 @@ int main(int argc, char** argv) { leaderTransferWait++; if (leaderTransferWait == 7) { - sTrace("begin leader transfer ..."); - int32_t ret = syncLeaderTransfer(rid); + if (leaderTransfer) { + sTrace("begin leader transfer ..."); + int32_t ret = syncLeaderTransfer(rid); + } } if (alreadySend < writeRecordNum) { From 54921550097c78bd3e769b5797897a4aa720cd8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 14:49:37 +0800 Subject: [PATCH 27/90] fix(query): fix syntax error on windows platform. --- source/libs/executor/src/sortoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 98a1aacbb6..2f2080d5fe 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -439,7 +439,7 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, pInfo->bufPageSize = getProperSortPageSize(rowSize); uint32_t numOfSources = taosArrayGetSize(pSortInfo); - numOfSources = MAX(2, numOfSources); + numOfSources = TMAX(2, numOfSources); pInfo->sortBufSize = numOfSources * pInfo->bufPageSize; From d41efb2cfb284be46997466601b929b0fc4527db Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 16 Jun 2022 15:02:57 +0800 Subject: [PATCH 28/90] fix rc issue --- source/libs/scheduler/src/schJob.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 3e3f3639d7..31536f413d 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -1488,13 +1488,15 @@ int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bo qDebug("QID:0x%" PRIx64 " job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); *job = pJob->refId; + if (!sync) { + pJob->userCb = SCH_EXEC_CB; + } + SCH_ERR_JRET(schLaunchJob(pJob)); if (sync) { SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); - } else { - pJob->userCb = SCH_EXEC_CB; } SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%"PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); From 7aa087a48d4c5ae48e597f27c35f06b9964e5d44 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 15:20:46 +0800 Subject: [PATCH 29/90] feat: redistribute vgroup --- include/util/taoserror.h | 8 +- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/mnode/impl/src/mndSync.c | 8 -- source/dnode/vnode/src/vnd/vnodeSvr.c | 102 --------------- source/dnode/vnode/src/vnd/vnodeSync.c | 166 +++++++++++++++++++++---- source/libs/sync/src/syncMain.c | 2 +- source/util/src/terror.c | 2 +- 7 files changed, 152 insertions(+), 138 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4f93ce9e4a..fcbe5cd878 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -410,10 +410,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) - -#define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x0910) -#define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0911) -#define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0912) +#define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) +#define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x090C) +#define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) +#define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x090E) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index e141e7dfb3..f277a5169d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -397,7 +397,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); - snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + snprintf(logBuf, sizeof(logBuf), "==mndProcessSyncMsg== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); static int64_t mndTick = 0; if (++mndTick % 10 == 1) { mTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 63708aef8a..e7b4489ed9 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -256,14 +256,6 @@ void mndSyncStart(SMnode *pMnode) { syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); syncStart(pMgmt->sync); mDebug("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); - - /* - if (pMgmt->standby) { - syncStartStandBy(pMgmt->sync); - } else { - syncStart(pMgmt->sync); - } - */ } void mndSyncStop(SMnode *pMnode) {} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c981f6cb99..d170931e7e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -300,108 +300,6 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - int32_t ret = 0; - - if (syncEnvIsStart()) { - SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); - assert(pSyncNode != NULL); - - ESyncState state = syncGetMyRole(pVnode->sync); - SyncTerm currentTerm = syncGetMyTerm(pVnode->sync); - - SMsgHead *pHead = pMsg->pCont; - - char logBuf[512] = {0}; - char *syncNodeStr = sync2SimpleStr(pVnode->sync); - snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); - static int64_t vndTick = 0; - if (++vndTick % 10 == 1) { - vTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); - } - syncRpcMsgLog2(logBuf, pMsg); - taosMemoryFree(syncNodeStr); - - SRpcMsg *pRpcMsg = pMsg; - - if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { - SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnPingCb(pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); - syncClientRequestDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); - - ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); - - } else if (pRpcMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) { - ret = syncSetStandby(pVnode->sync); - vInfo("vgId:%d, set standby result:0x%x rid:%" PRId64, pVnode->config.vgId, ret, pVnode->sync); - SRpcMsg rsp = {.code = ret, .info = pMsg->info}; - tmsgSendRsp(&rsp); - } else { - vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); - ret = -1; - } - - syncNodeRelease(pSyncNode); - } else { - vError("==vnodeProcessSyncReq== error syncEnv stop"); - ret = -1; - } - - if (ret != 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - } - return ret; -} - static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder coder; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 25661d2e69..8662c8369b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -50,6 +50,33 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) { } } +static int32_t vnodeSetStandBy(SVnode *pVnode) { + vInfo("vgId:%d, start to set standby", TD_VID(pVnode)); + + if (syncSetStandby(pVnode->sync) == 0) { + vInfo("vgId:%d, set standby success", TD_VID(pVnode)); + return 0; + } else if (terrno != TSDB_CODE_SYN_IS_LEADER) { + vError("vgId:%d, failed to set standby since %s", TD_VID(pVnode), terrstr()); + return -1; + } + + if (syncLeaderTransfer(pVnode->sync) != 0) { + vError("vgId:%d, failed to transfer leader since:%s", TD_VID(pVnode), terrstr()); + return -1; + } else { + vInfo("vgId:%d, transfer leader success", TD_VID(pVnode)); + } + + if (syncSetStandby(pVnode->sync) == 0) { + vInfo("vgId:%d, set standby success", TD_VID(pVnode)); + return 0; + } else { + vError("vgId:%d, failed to set standby since %s", TD_VID(pVnode), terrstr()); + return -1; + } +} + static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) { SAlterVnodeReq req = {0}; if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) { @@ -74,14 +101,19 @@ static int32_t vnodeProcessAlterReplicaReq(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = syncPropose(pVnode->sync, &rpcMsg, false); if (code != 0) { - vDebug("vgId:%d, failed to propose reconfig msg since %s", TD_VID(pVnode), terrstr()); - if (syncLeaderTransfer(pVnode->sync) != 0) { - vError("vgId:%d, failed to transfer leader since %s", TD_VID(pVnode), terrstr()); - } else { - vDebug("vgId:%d, transfer leader success, propose reconfig config again", TD_VID(pVnode)); + if (terrno != 0) code = terrno; + + vInfo("vgId:%d, failed to propose reconfig msg since %s", TD_VID(pVnode), terrstr()); + if (terrno == TSDB_CODE_SYN_IS_LEADER) { + if (syncLeaderTransfer(pVnode->sync) != 0) { + vError("vgId:%d, failed to transfer leader since %s", TD_VID(pVnode), terrstr()); + } else { + vInfo("vgId:%d, transfer leader success", TD_VID(pVnode)); + } } } + terrno = code; return code; } @@ -108,7 +140,6 @@ void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { if (code == 0) { vnodeAccumBlockMsg(pVnode, pMsg->msgType); - } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { SEpSet newEpSet = {0}; syncGetEpSet(pVnode->sync, &newEpSet); @@ -170,6 +201,108 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { } } +int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + int32_t ret = 0; + + if (syncEnvIsStart()) { + SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); + assert(pSyncNode != NULL); + + ESyncState state = syncGetMyRole(pVnode->sync); + SyncTerm currentTerm = syncGetMyTerm(pVnode->sync); + + SMsgHead *pHead = pMsg->pCont; + + char logBuf[512] = {0}; + char *syncNodeStr = sync2SimpleStr(pVnode->sync); + snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); + static int64_t vndTick = 0; + if (++vndTick % 10 == 1) { + vTrace("sync trace msg:%s, %s", TMSG_INFO(pMsg->msgType), syncNodeStr); + } + syncRpcMsgLog2(logBuf, pMsg); + taosMemoryFree(syncNodeStr); + + SRpcMsg *pRpcMsg = pMsg; + + if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { + SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnPingCb(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); + syncClientRequestDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); + assert(pSyncMsg != NULL); + + ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + + } else if (pRpcMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) { + ret = vnodeSetStandBy(pVnode); + if (ret != 0 && terrno != 0) ret = terrno; + SRpcMsg rsp = {.code = ret, .info = pMsg->info}; + tmsgSendRsp(&rsp); + } else { + vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); + ret = -1; + } + + syncNodeRelease(pSyncNode); + } else { + vError("==vnodeProcessSyncReq== error syncEnv stop"); + ret = -1; + } + + if (ret != 0) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + } + return ret; +} + static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); if (code != 0) { @@ -258,17 +391,17 @@ static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } -int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void **ppReader) { return 0; } +static int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void **ppReader) { return 0; } -int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) { return 0; } +static int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) { return 0; } -int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { return 0; } +static int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) { return 0; } -int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void **ppWriter) { return 0; } +static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void **ppWriter) { return 0; } -int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { return 0; } +static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { return 0; } -int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { return 0; } +static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { return 0; } static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); @@ -279,7 +412,6 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpGetSnapshot = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = NULL; pFsm->FpReConfigCb = vnodeSyncReconfig; - pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; pFsm->FpSnapshotDoRead = vnodeSnapshotDoRead; @@ -292,7 +424,6 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncInfo syncInfo = { - .isStandBy = false, .snapshotEnable = false, .vgId = pVnode->config.vgId, .isStandBy = pVnode->config.standby, @@ -321,13 +452,6 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { void vnodeSyncStart(SVnode *pVnode) { syncSetMsgCb(pVnode->sync, &pVnode->msgCb); syncStart(pVnode->sync); - /* - if (pVnode->config.standby) { - syncStartStandBy(pVnode->sync); - } else { - syncStart(pVnode->sync); - } - */ } void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 85bdf18ab4..03a4f7b2b3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -156,7 +156,7 @@ int32_t syncSetStandby(int64_t rid) { if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + terrno = TSDB_CODE_SYN_IS_LEADER; sError("failed to set standby since it is leader, rid:%" PRId64, rid); return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index fe040df6fc..596aab0e09 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -412,7 +412,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_SIGNATURE, "Mismatched signature" TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CHECKSUM, "Invalid msg checksum") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, "Invalid msg length") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") - +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_IS_LEADER, "Sync is leader") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync not leader") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_ONE_REPLICA, "Sync one replica") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_IN_NEW_CONFIG, "Sync not in new config") From 855f1842c2760e038c8f3cda43b1ef1455e7a12c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Thu, 16 Jun 2022 15:28:52 +0800 Subject: [PATCH 30/90] ci: allow to run specified cases in windows full test --- tests/system-test/test-all.bat | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 0929b1fc6e..275cbeebbb 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -5,7 +5,12 @@ set /a a=0 if %1 == full ( echo Windows Taosd Full Test set /a exitNum=0 - for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( + del /Q /F failed.txt + set caseFile="fulltest.bat" + if not "%2" == "" ( + set caseFile="%2" + ) + for /F "usebackq tokens=*" %%i in (!caseFile!) do ( for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" ( set /a a+=1 echo !a! Processing %%i @@ -13,7 +18,7 @@ if %1 == full ( set time1=!_timeTemp! echo Start at !time! call %%i ARG1 > result_!a!.txt 2>error_!a!.txt - if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 ) else ( call :colorEcho 0a "Success" &echo. ) + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) ) ) exit !exitNum! @@ -77,4 +82,4 @@ for %%a in (%tt%) do ( set /a index=index+1 ) set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% -goto :eof \ No newline at end of file +goto :eof From dbef3dfc55d19e1d3e182ef61f23629fe7261eff Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 16 Jun 2022 15:34:33 +0800 Subject: [PATCH 31/90] fix: not submit for rsma without rollup --- source/common/src/tdatablock.c | 42 ++++++++++++++++---------- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/sma/smaRollup.c | 14 +++++++-- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 5a2aaed74e..3ec1b98b16 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1583,6 +1583,11 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t rowSize = pDataBlock->info.rowSize; int64_t groupId = pDataBlock->info.groupId; + if (colNum <= 1) { + // invalid if only with TS col + continue; + } + if (rb.nCols != colNum) { tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); } @@ -1679,23 +1684,28 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks msgLen += pSubmitBlk->dataLen; } - (*pReq)->length = msgLen; + if (numOfBlks > 0) { + (*pReq)->length = msgLen; - (*pReq)->header.vgId = htonl(vgId); - (*pReq)->header.contLen = htonl(msgLen); - (*pReq)->length = (*pReq)->header.contLen; - (*pReq)->numOfBlocks = htonl(numOfBlks); - SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); - while (numOfBlks--) { - int32_t dataLen = blk->dataLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->padding = htonl(blk->padding); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); + (*pReq)->header.vgId = htonl(vgId); + (*pReq)->header.contLen = htonl(msgLen); + (*pReq)->length = (*pReq)->header.contLen; + (*pReq)->numOfBlocks = htonl(numOfBlks); + SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); + while (numOfBlks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->suid = htobe64(blk->suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } + } else { + // no valid rows + taosMemoryFreeClear(*pReq); } return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2fd0815181..553c6a40ab 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -260,6 +260,7 @@ struct SSma { #define SMA_CFG(s) (&(s)->pVnode->config) #define SMA_TSDB_CFG(s) (&(s)->pVnode->config.tsdbCfg) +#define SMA_RETENTION(s) ((SRetention *)&(s)->pVnode->config.tsdbCfg.retentions) #define SMA_LOCKED(s) ((s)->locked) #define SMA_META(s) ((s)->pVnode->pMeta) #define SMA_VID(s) TD_VID((s)->pVnode) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index b2dcce8f4c..0c372dfa70 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -400,22 +400,24 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } if (taosArrayGetSize(pResult) > 0) { -#if 1 +#if 0 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, level); blockDebugShowData(pResult, flag); #endif STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; - if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) != 0) { + if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) { taosArrayDestroy(pResult); return TSDB_CODE_FAILED; } - if (tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) { + + if (pReq && tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) < 0) { taosArrayDestroy(pResult); taosMemoryFreeClear(pReq); return TSDB_CODE_FAILED; } + taosMemoryFreeClear(pReq); } else { smaDebug("vgId:%d, no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); @@ -469,6 +471,12 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } + SRetention *pRetention = SMA_RETENTION(pSma); + if (!RETENTION_VALID(pRetention + 1)) { + // return directly if retention level 1 is invalid + return TSDB_CODE_SUCCESS; + } + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { STbUidStore uidStore = {0}; tdFetchSubmitReqSuids(pMsg, &uidStore); From 9a9200d28b93ae286f19b530688c3d28dfce94e6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 15:38:40 +0800 Subject: [PATCH 32/90] refactor(sync): add restore finish when become leader again --- source/libs/sync/src/syncAppendEntries.c | 7 +- source/libs/sync/src/syncAppendEntriesReply.c | 13 +- source/libs/sync/src/syncCommit.c | 5 +- source/libs/sync/src/syncMain.c | 70 +++++----- source/libs/sync/src/syncRaftLog.c | 16 ++- source/libs/sync/src/syncRespMgr.c | 19 +-- source/libs/sync/src/syncSnapshot.c | 131 ++++++++++-------- 7 files changed, 147 insertions(+), 114 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 08a4081ad3..89b4761212 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -713,7 +713,8 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - sDebug("vgId:%d sync event log truncate, from %ld to %ld", ths->vgId, delBegin, delEnd); + sDebug("vgId:%d sync event currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, ths->pRaftStore->currentTerm, + delBegin, delEnd); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); return code; @@ -994,8 +995,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs SyncIndex commitEnd = snapshot.lastApplyIndex; ths->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", ths->vgId, commitBegin, - commitEnd, syncUtilState2String(ths->state)); + sDebug("vgId:%d sync event currentTerm:%lu commit by snapshot from index:%ld to index:%ld, %s", ths->vgId, + ths->pRaftStore->currentTerm, commitBegin, commitEnd, syncUtilState2String(ths->state)); } SyncIndex beginIndex = ths->commitIndex + 1; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index fbe3319675..5290d7d28e 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -190,18 +190,19 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (gRaftDetailLog) { char* s = snapshotSender2Str(pSender); sDebug( - "vgId:%d sync event snapshot send to %s:%d start sender first time, lastApplyIndex:%ld lastApplyTerm:%lu " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " + "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu " "sender:%s", - ths->vgId, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - pSender->snapshot.lastConfigIndex, pSender->privateTerm, s); + ths->vgId, ths->pRaftStore->currentTerm, host, port, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, s); taosMemoryFree(s); } else { sDebug( - "vgId:%d sync event snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " "lastApplyTerm:%lu lastConfigIndex:%ld privateTerm:%lu", - ths->vgId, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, - pSender->snapshot.lastConfigIndex, pSender->privateTerm); + ths->vgId, ths->pRaftStore->currentTerm, host, port, pSender->snapshot.lastApplyIndex, + pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 3424fac5e7..efdb5b24ce 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -56,8 +56,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex commitEnd = snapshot.lastApplyIndex; pSyncNode->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", pSyncNode->vgId, - pSyncNode->commitIndex, snapshot.lastApplyIndex, syncUtilState2String(pSyncNode->state)); + sDebug("vgId:%d sync event currentTerm:%lu commit by snapshot from index:%ld to index:%ld, %s", pSyncNode->vgId, + pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, snapshot.lastApplyIndex, + syncUtilState2String(pSyncNode->state)); } // update commit index diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 5e3c1adf32..05a41daa9e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -562,7 +562,8 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return -1; } assert(rid == pSyncNode->rid); - sDebug("vgId:%d sync event propose msgType:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), pMsg->msgType); + sDebug("vgId:%d sync event currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), pMsg->msgType); ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -571,7 +572,8 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; - sDebug("vgId:%d sync event propose msgType:%s,%d", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), pMsg->msgType); + sDebug("vgId:%d sync event currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), pMsg->msgType); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -604,8 +606,6 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; - sDebug("vgId:%d sync event sync open", pSyncInfo->vgId); - SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); @@ -819,6 +819,8 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // snapshot meta // pSyncNode->sMeta.lastConfigIndex = -1; + sDebug("vgId:%d sync event currentTerm:%lu sync open", pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm); + return pSyncNode; } @@ -864,7 +866,7 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodeClose(SSyncNode* pSyncNode) { - sDebug("vgId:%d sync event sync close", pSyncNode->vgId); + sDebug("vgId:%d sync event currentTerm:%lu sync close", pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm); int32_t ret; assert(pSyncNode != NULL); @@ -1303,8 +1305,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { oldSenders[i] = (pSyncNode->senders)[i]; - sDebug("vgId:%d sync event save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, i, oldSenders[i], - oldSenders[i]->privateTerm); + sDebug("vgId:%d sync event currentTerm:%lu save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, + pSyncNode->pRaftStore->currentTerm, i, oldSenders[i], oldSenders[i]->privateTerm); if (gRaftDetailLog) { ; } @@ -1356,8 +1358,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event reset sender for %lu, newIndex:%d, %s:%d, %p, privateTerm:%lu", pSyncNode->vgId, - (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j], oldSenders[j]->privateTerm); + sDebug("vgId:%d sync event currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, privateTerm:%lu", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, (pSyncNode->replicasId)[i].addr, i, host, port, + oldSenders[j], oldSenders[j]->privateTerm); (pSyncNode->senders)[i] = oldSenders[j]; oldSenders[j] = NULL; reset = true; @@ -1365,8 +1368,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex // reset replicaIndex int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; (pSyncNode->senders)[i]->replicaIndex = i; - sDebug("vgId:%d sync event udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", pSyncNode->vgId, - oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); + sDebug("vgId:%d sync event currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, oldreplicaIndex, i, host, port, + (pSyncNode->senders)[i], reset); } } } @@ -1375,8 +1379,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sDebug("vgId:%d sync event create new sender %p replicaIndex:%d, privateTerm:%lu", pSyncNode->vgId, - (pSyncNode->senders)[i], i, (pSyncNode->senders)[i]->privateTerm); + sDebug("vgId:%d sync event currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, (pSyncNode->senders)[i], i, + (pSyncNode->senders)[i]->privateTerm); } } @@ -1384,7 +1389,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { snapshotSenderDestroy(oldSenders[i]); - sDebug("vgId:%d sync event delete old sender %p replicaIndex:%d", pSyncNode->vgId, oldSenders[i], i); + sDebug("vgId:%d sync event currentTerm:%lu delete old sender %p replicaIndex:%d", pSyncNode->vgId, + pSyncNode->pRaftStore->currentTerm, oldSenders[i], i); oldSenders[i] = NULL; } } @@ -1454,9 +1460,11 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { } void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become follower, isStandBy:%d, currentTerm:%lu, replicaNum:%d, restoreFinish:%d, %s", - pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftStore->currentTerm, pSyncNode->replicaNum, - pSyncNode->restoreFinish, debugStr); + sDebug( + "vgId:%d sync event currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " + "restoreFinish:%d, %s", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->restoreFinish, debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1493,8 +1501,8 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // reset restoreFinish pSyncNode->restoreFinish = false; - sDebug("vgId:%d sync event become leader, isStandBy:%d, currentTerm:%lu, replicaNum:%d, restoreFinish:%d, %s", - pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftStore->currentTerm, pSyncNode->replicaNum, + sDebug("vgId:%d sync event currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, restoreFinish:%d, %s", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->restoreFinish, debugStr); // state change @@ -2069,21 +2077,13 @@ const char* syncStr(ESyncState state) { static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - /* - char host[128]; - uint16_t port; - syncUtilU642Addr(pSyncLeaderTransfer->newLeaderId.addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event, maybe leader transfer to %s:%d %lu", ths->vgId, host, port, - pSyncLeaderTransfer->newLeaderId.addr); - */ - - sDebug("vgId:%d sync event begin leader transfer", ths->vgId); + sDebug("vgId:%d sync event currentTerm:%lu begin leader transfer", ths->vgId, ths->pRaftStore->currentTerm); if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { - sDebug("vgId:%d sync event maybe leader transfer to %s:%d %lu", ths->vgId, - pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, - pSyncLeaderTransfer->newLeaderId.addr); + sDebug("vgId:%d sync event currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, + ths->pRaftStore->currentTerm, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, + pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); // reset elect timer now! int32_t electMS = 1; @@ -2205,8 +2205,8 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; - sDebug("vgId:%d sync event commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, beginIndex, - endIndex, syncUtilState2String(state)); + sDebug("vgId:%d sync event currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, + ths->pRaftStore->currentTerm, beginIndex, endIndex, syncUtilState2String(state)); // execute fsm if (ths->pFsm != NULL) { @@ -2254,8 +2254,8 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sDebug("vgId:%d sync event restore finish, %s, index:%ld", ths->vgId, syncUtilState2String(ths->state), - pEntry->index); + sDebug("vgId:%d sync event currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, + ths->pRaftStore->currentTerm, syncUtilState2String(ths->state), pEntry->index); } } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 79ce84abe9..31ac44fa81 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -15,6 +15,7 @@ #include "syncRaftLog.h" #include "syncRaftCfg.h" +#include "syncRaftStore.h" #include "wal.h" // refactor, log[0 .. n] ==> log[m .. n] @@ -162,10 +163,10 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); - sDebug("vgId:%d sync event write index:%ld, %s, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", - pData->pSyncNode->vgId, pEntry->index, syncUtilState2String(pData->pSyncNode->state), - pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), pEntry->msgType, - TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + sDebug("vgId:%d sync event currentTerm:%lu write index:%ld, %s, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", + pData->pSyncNode->vgId, pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, + syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftCfg->isStandBy, + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); return code; } @@ -321,7 +322,12 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { walFsync(pWal, true); - sDebug("sync event old write wal: %ld", pEntry->index); + sDebug( + "vgId:%d sync event currentTerm:%lu old write index:%ld, %s, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", + pData->pSyncNode->vgId, pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, + syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), + pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + return code; } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index adc67b5d8d..5b127793d6 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -14,6 +14,7 @@ */ #include "syncRespMgr.h" +#include "syncRaftStore.h" SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr)); @@ -45,9 +46,9 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + sDebug("vgId:%d sync event currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, + pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, + pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return keyCode; @@ -70,9 +71,9 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event resp mgr get, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + sDebug("vgId:%d sync event currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, + index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return 1; // get one object @@ -89,9 +90,9 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event resp mgr get and del, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + sDebug("vgId:%d sync event currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, + index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); taosThreadMutexUnlock(&(pObj->mutex)); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 545ea953f8..d3785060d4 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -141,18 +141,22 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " + "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " + "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -279,25 +283,31 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " + "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " + "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm); } } else { sDebug( - "vgId:%d sync event snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld lastApplyTerm:%lu " + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld " + "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -328,12 +338,15 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); - sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu send msg:%s", - pSender->pSyncNode->vgId, host, port, pSender->seq, pSender->ack, pSender->privateTerm, msgStr); + sDebug( + "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu send msg:%s", + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { - sDebug("vgId:%d sync event snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu", pSender->pSyncNode->vgId, - host, port, pSender->seq, pSender->ack, pSender->privateTerm); + sDebug("vgId:%d sync event currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu", + pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, + pSender->ack, pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -485,7 +498,7 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { pReceiver->start = false; if (apply) { - ++(pReceiver->privateTerm); + // ++(pReceiver->privateTerm); } if (gRaftDetailLog) { @@ -566,17 +579,17 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, - pReceiver->privateTerm, msgStr); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld privateTerm:%lu", - pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, - pReceiver->privateTerm); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -612,14 +625,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { bool isDrop; if (IamInNew) { - sDebug("vgId:%d sync event update config by snapshot, lastIndex:%ld, lastTerm:%lu, lastConfigIndex:%ld ", - pSyncNode->vgId, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + sDebug( + "vgId:%d sync event currentTerm:%lu update config by snapshot, lastIndex:%ld, lastTerm:%lu, " + "lastConfigIndex:%ld ", + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex); syncNodeUpdateConfig(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex, &isDrop); } else { sDebug( - "vgId:%d sync event do not update config by snapshot, I am not in newCfg, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu do not update config by snapshot, I am not in newCfg, " + "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld ", - pSyncNode->vgId, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex); } // change isStandBy to normal @@ -644,19 +662,20 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); sDebug( - "vgId:%d sync event snapshot recv from %s:%d finish, update log begin index:%ld, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu, raft log:%s", - pSyncNode->vgId, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - snapshot.lastConfigIndex, pReceiver->privateTerm, logSimpleStr); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pMsg->lastIndex + 1, + snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, pReceiver->privateTerm, + logSimpleStr); taosMemoryFree(logSimpleStr); } else { sDebug( - "vgId:%d sync event snapshot recv from %s:%d finish, update log begin index:%ld, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu", - pSyncNode->vgId, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, - snapshot.lastConfigIndex, pReceiver->privateTerm); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pMsg->lastIndex + 1, + snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, pReceiver->privateTerm); } pReceiver->pWriter = NULL; @@ -667,17 +686,17 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex, pReceiver->privateTerm); + pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -692,18 +711,20 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv " "msg:%s", - pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pReceiver->pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex, pReceiver->privateTerm); + pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, + pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { @@ -723,17 +744,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, - pReceiver->privateTerm, msgStr); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pSyncNode->vgId, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, - pReceiver->privateTerm); + pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else { From 528d7d7248cbafb734aec558368cacbcaa832551 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 16 Jun 2022 15:46:51 +0800 Subject: [PATCH 33/90] os: add dll check --- include/os/osSysinfo.h | 1 + source/client/src/clientHb.c | 4 +++- source/libs/catalog/src/ctgCache.c | 4 +++- source/os/src/osSysinfo.c | 16 ++++++++++++++++ source/util/src/tcache.c | 6 +++++- 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index c009bcf350..4ec2e2884e 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -68,6 +68,7 @@ typedef struct { } SysNameInfo; SysNameInfo taosGetSysNameInfo(); +bool taosCheckCurrentInDll(); #ifdef __cplusplus } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 5db27ae438..c450e9351f 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -588,7 +588,9 @@ void hbThreadFuncUnexpectedStopped(void) { static void *hbThreadFunc(void *param) { setThreadName("hb"); #ifdef WINDOWS - atexit(hbThreadFuncUnexpectedStopped); + if (taosCheckCurrentInDll()) { + atexit(hbThreadFuncUnexpectedStopped); + } #endif while (1) { int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 277516686b..0948c01270 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1938,7 +1938,9 @@ void ctgCleanupCacheQueue(void) { void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); #ifdef WINDOWS - atexit(ctgUpdateThreadUnexpectedStopped); + if (taosCheckCurrentInDll()) { + atexit(ctgUpdateThreadUnexpectedStopped); + } #endif qInfo("catalog update thread started"); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index ace870853f..4981f7dc26 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -945,3 +945,19 @@ SysNameInfo taosGetSysNameInfo() { return info; #endif } + + +bool taosCheckCurrentInDll() { +#ifdef WINDOWS + MEMORY_BASIC_INFORMATION mbi; + char path[PATH_MAX] = {0}; + GetModuleFileName(((VirtualQuery(taosCheckCurrentInDll,&mbi,sizeof(mbi)) != 0) ? (HMODULE)mbi.AllocationBase : NULL), path, PATH_MAX); + int strLastIndex = strlen(path); + if ((path[strLastIndex-3] == 'd' || path[strLastIndex-3] == 'D') && (path[strLastIndex-2] == 'l' || path[strLastIndex-2] == 'L') && (path[strLastIndex-1] == 'l' || path[strLastIndex-1] == 'L')) { + return true; + } + return false; +#else + return false; +#endif +} \ No newline at end of file diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 10a5475555..1a716752a9 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -829,7 +829,11 @@ void *taosCacheTimedRefresh(void *handle) { const int32_t SLEEP_DURATION = 500; // 500 ms int64_t count = 0; - atexit(taosCacheRefreshWorkerUnexpectedStopped); +#ifdef WINDOWS + if (taosCheckCurrentInDll()) { + atexit(taosCacheRefreshWorkerUnexpectedStopped); + } +#endif while (1) { taosMsleep(SLEEP_DURATION); From 14b6523bd34981a1cd7b86dbddc711e278101a32 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 15:56:25 +0800 Subject: [PATCH 34/90] refactor: show rpc response code --- source/libs/transport/src/transCli.c | 4 ++-- source/libs/transport/src/transSvr.c | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index fbf4d6aa9f..3d570e0860 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -324,9 +324,9 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, + tDebug("%s cli conn %p %s received from %s:%d, local info:%s:%d, msg size:%d code:0x%x", pTransInst->label, conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), - taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen); + taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tTrace("except, server continue send while cli ignore it"); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 020435d076..b6e3fd2676 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -285,13 +285,14 @@ static void uvHandleReq(SSvrConn* pConn) { } if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), - ntohs(pConn->localAddr.sin_port), transMsg.contLen); - } else { - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, + tDebug("server conn %p %s received from %s:%d, local info:%s:%d, msg size:%d code:0x%x", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port), transMsg.contLen, pHead->noResp); + taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port), transMsg.contLen, transMsg.code); + } else { + tDebug("server conn %p %s received from %s:%d, local info:%s:%d, msg size:%d, resp:%d code:0x%x", pConn, + TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port), transMsg.contLen, pHead->noResp, + transMsg.code); // no ref here } From 928f45709dfce0946ce9dc76b7162ba7cd5718df Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 16:01:22 +0800 Subject: [PATCH 35/90] feat(stream): drop stream --- include/common/tmsg.h | 4 +- include/util/tdef.h | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 51 +++-- source/dnode/mnode/impl/inc/mndStb.h | 6 +- source/dnode/mnode/impl/inc/mndStream.h | 2 + source/dnode/mnode/impl/src/mndDef.c | 39 +--- source/dnode/mnode/impl/src/mndScheduler.c | 12 +- source/dnode/mnode/impl/src/mndSma.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 31 ++- source/dnode/mnode/impl/src/mndStream.c | 244 +++++++++++++++++++-- 10 files changed, 295 insertions(+), 100 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7c0e8206e1..01d99ac705 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1519,7 +1519,7 @@ typedef struct { #define STREAM_TRIGGER_MAX_DELAY 3 typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; + char name[TSDB_STREAM_FNAME_LEN]; char sourceDB[TSDB_DB_FNAME_LEN]; char targetStbFullName[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -1539,7 +1539,7 @@ int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStrea void tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq); typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; + char name[TSDB_STREAM_FNAME_LEN]; int64_t streamId; char* sql; char* executorMsg; diff --git a/include/util/tdef.h b/include/util/tdef.h index 4798a9375c..a48e823bad 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -210,7 +210,7 @@ typedef enum ELogicConditionType { #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_STREAM_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) #define TSDB_COL_NAME_LEN 65 diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 35cbf178ec..db18f87830 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -539,27 +539,38 @@ typedef struct { } SMqRebOutputObj; typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - char sourceDb[TSDB_DB_FNAME_LEN]; - char targetDb[TSDB_DB_FNAME_LEN]; - char targetSTbName[TSDB_TABLE_FNAME_LEN]; - int64_t targetStbUid; - int64_t createTime; - int64_t updateTime; - int64_t uid; - int64_t dbUid; - int32_t version; - int32_t vgNum; - SRWLatch lock; - int8_t status; - int8_t createdBy; // STREAM_CREATED_BY__USER or SMA - int32_t fixedSinkVgId; // 0 for shuffle - SVgObj fixedSinkVg; - int64_t smaId; // 0 for unused - int8_t trigger; - int64_t triggerParam; - int64_t watermark; + char name[TSDB_STREAM_FNAME_LEN]; + // ctl + SRWLatch lock; + // create info + int64_t createTime; + int64_t updateTime; + int32_t version; + // TODO remove it + int64_t smaId; // 0 for unused + // info + int64_t uid; + int8_t status; + // TODO remove it + int32_t vgNum; + // config + int8_t dropPolicy; + int8_t trigger; + int64_t triggerParam; + int64_t watermark; + // source and target + int64_t sourceDbUid; + int64_t targetDbUid; + char sourceDb[TSDB_DB_FNAME_LEN]; + char targetDb[TSDB_DB_FNAME_LEN]; + char targetSTbName[TSDB_TABLE_FNAME_LEN]; + int64_t targetStbUid; + // assigned when scheduling + int32_t fixedSinkVgId; // 0 for shuffle + SVgObj fixedSinkVg; + // transformation char* sql; + char* ast; char* physicalPlan; SArray* tasks; // SArray> SSchemaWrapper outputSchema; diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index d3de4e3b3f..44a7fdadde 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -27,8 +27,7 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); SSdbRaw *mndStbActionEncode(SStbObj *pStb); -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbs, int32_t numOfStbs, void **ppRsp, - int32_t *pRspLen); +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbs, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate); @@ -36,6 +35,9 @@ SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName); int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreate, SDbObj *pDb); int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); +void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); +void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b5d22cb7a5..65017027dc 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -33,6 +33,8 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans); +int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 6eac91bb43..b9bd1e4298 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -19,7 +19,6 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { int32_t sz = 0; - /*int32_t outputNameSz = 0;*/ if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; @@ -28,10 +27,12 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->vgNum) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1; if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1; if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; @@ -57,17 +58,6 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { } if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; - -#if 0 - if (pObj->ColAlias != NULL) { - outputNameSz = taosArrayGetSize(pObj->ColAlias); - } - if (tEncodeI32(pEncoder, outputNameSz) < 0) return -1; - for (int32_t i = 0; i < outputNameSz; i++) { - char *name = taosArrayGetP(pObj->ColAlias, i); - if (tEncodeCStr(pEncoder, name) < 0) return -1; - } -#endif return pEncoder->pos; } @@ -80,10 +70,12 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->vgNum) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1; if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; @@ -112,21 +104,6 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { } if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; -#if 0 - int32_t outputNameSz; - if (tDecodeI32(pDecoder, &outputNameSz) < 0) return -1; - if (outputNameSz != 0) { - pObj->ColAlias = taosArrayInit(outputNameSz, sizeof(void *)); - if (pObj->ColAlias == NULL) { - return -1; - } - } - for (int32_t i = 0; i < outputNameSz; i++) { - char *name; - if (tDecodeCStrAlloc(pDecoder, &name) < 0) return -1; - taosArrayPush(pObj->ColAlias, &name); - } -#endif return 0; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index ffddab3a91..c53e97cd53 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -131,7 +131,7 @@ int32_t mndPersistTaskDeployReq(STrans* pTrans, SStreamTask* pTask, const SEpSet int32_t mndAddSinkToTask(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, SStreamTask* pTask) { pTask->dispatchType = TASK_DISPATCH__NONE; // sink - if (pStream->createdBy == STREAM_CREATED_BY__SMA) { + if (pStream->smaId != 0) { pTask->sinkType = TASK_SINK__SMA; pTask->smaSink.smaId = pStream->smaId; } else { @@ -275,7 +275,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb pTask->execType = TASK_EXEC__NONE; // sink - if (pStream->createdBy == STREAM_CREATED_BY__SMA) { + if (pStream->smaId != 0) { pTask->sinkType = TASK_SINK__SMA; pTask->smaSink.smaId = pStream->smaId; } else { @@ -321,7 +321,7 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pTask->execType = TASK_EXEC__NONE; // sink - if (pStream->createdBy == STREAM_CREATED_BY__SMA) { + if (pStream->smaId != 0) { pTask->sinkType = TASK_SINK__SMA; pTask->smaSink.smaId = pStream->smaId; } else { @@ -399,7 +399,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { // exec pFinalTask->execType = TASK_EXEC__PIPE; - SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->dbUid); + SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid); if (mndAssignTaskToVg(pMnode, pTrans, pFinalTask, plan, pVgroup) < 0) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); @@ -420,7 +420,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); if (pIter == NULL) break; - if (pVgroup->dbUid != pStream->dbUid) { + if (pVgroup->dbUid != pStream->sourceDbUid) { sdbRelease(pSdb, pVgroup); continue; } @@ -463,7 +463,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SVgObj* pVgroup; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); if (pIter == NULL) break; - if (pVgroup->dbUid != pStream->dbUid) { + if (pVgroup->dbUid != pStream->sourceDbUid) { sdbRelease(pSdb, pVgroup); continue; } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0f6bb74d9a..54fd0fe867 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -520,6 +520,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN); smaObj.createdTime = taosGetTimestampMs(); smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + ASSERT(smaObj.uid != 0); char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0}; snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "td.tsma.rst.tb.%s", pCreate->name); memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN); @@ -556,10 +557,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); - streamObj.dbUid = pDb->uid; + streamObj.sourceDbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; - streamObj.createdBy = STREAM_CREATED_BY__SMA; streamObj.smaId = smaObj.uid; streamObj.watermark = 0; streamObj.trigger = STREAM_TRIGGER_AT_ONCE; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d4a38dd17b..b83aa34a3a 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -28,7 +28,6 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#include "mndSma.h" #include "tname.h" #define STB_VER_NUMBER 1 @@ -1271,7 +1270,8 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return 0; } -static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, + int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); @@ -1672,7 +1672,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + for (int32_t i = 0; i < numOfStbs; ++i) { SSTableVersion *pStbVersion = &pStbVersions[i]; pStbVersion->suid = be64toh(pStbVersion->suid); @@ -1681,7 +1681,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t pStbVersion->smaVer = ntohl(pStbVersion->smaVer); STableMetaRsp metaRsp = {0}; - int32_t smaVer = 0; + int32_t smaVer = 0; mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; @@ -1697,15 +1697,15 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t } if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) { - bool exist = false; - char tbFName[TSDB_TABLE_FNAME_LEN]; + bool exist = false; + char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); if (NULL == indexRsp.pIndex) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { @@ -1769,16 +1769,23 @@ int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { return 0; } -static void mndExtractTableName(char *tableId, char *name) { +void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + tNameGetFullDbName(&name, dst); +} + +void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { int32_t pos = -1; int32_t num = 0; - for (pos = 0; tableId[pos] != 0; ++pos) { - if (tableId[pos] == TS_PATH_DELIMITER[0]) num++; + for (pos = 0; stbFullName[pos] != 0; ++pos) { + if (stbFullName[pos] == TS_PATH_DELIMITER[0]) num++; if (num == 2) break; } if (num == 2) { - strcpy(name, tableId + pos + 1); + tstrncpy(dst, stbFullName + pos + 1, dstSize); } } @@ -1808,7 +1815,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SName name = {0}; char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - mndExtractTableName(pStb->name, &stbName[VARSTR_HEADER_SIZE]); + mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 491835fc15..de0b25b1dc 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -196,7 +196,8 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { } static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { - if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { + if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->sourceDB[0] == 0 || + pCreate->targetStbFullName[0] == 0) { terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION; return -1; } @@ -232,6 +233,114 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 return code; } +static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SCMCreateStreamReq *pCreate) { + SNode *pAst = NULL; + + mDebug("stream:%s to create", pCreate->name); + memcpy(pObj->name, pCreate->name, TSDB_STREAM_FNAME_LEN); + pObj->createTime = taosGetTimestampMs(); + pObj->updateTime = pObj->createTime; + pObj->version = 1; + pObj->smaId = 0; + + pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name)); + pObj->status = 0; + + // TODO + pObj->dropPolicy = 0; + pObj->trigger = pCreate->triggerType; + pObj->triggerParam = pCreate->maxDelay; + pObj->watermark = pCreate->watermark; + + memcpy(pObj->sourceDb, pCreate->sourceDB, TSDB_DB_FNAME_LEN); + SDbObj *pSourceDb = mndAcquireDb(pMnode, pCreate->sourceDB); + if (pSourceDb == NULL) { + /*ASSERT(0);*/ + mDebug("stream:%s failed to create, source db %s not exist", pCreate->name, pObj->sourceDb); + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + return -1; + } + pObj->sourceDbUid = pSourceDb->uid; + + memcpy(pObj->targetSTbName, pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN); + + SDbObj *pTargetDb = mndAcquireDbByStb(pMnode, pObj->targetSTbName); + if (pTargetDb == NULL) { + mDebug("stream:%s failed to create, target db %s not exist", pCreate->name, pObj->targetDb); + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + return -1; + } + tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); + + pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + pObj->targetDbUid = pTargetDb->uid; + + pObj->sql = pCreate->sql; + pObj->ast = pCreate->ast; + + pCreate->sql = NULL; + pCreate->ast = NULL; + + // deserialize ast + if (nodesStringToNode(pObj->ast, &pAst) < 0) { + /*ASSERT(0);*/ + goto FAIL; + } + + // extract output schema from ast + if (qExtractResultSchema(pAst, (int32_t *)&pObj->outputSchema.nCols, &pObj->outputSchema.pSchema) != 0) { + /*ASSERT(0);*/ + goto FAIL; + } + + SQueryPlan *pPlan = NULL; + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + .triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger, + .watermark = pObj->watermark, + }; + + // using ast and param to build physical plan + if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { + /*ASSERT(0);*/ + goto FAIL; + } + + // save physcial plan + if (nodesNodeToString((SNode *)pPlan, false, &pObj->physicalPlan, NULL) != 0) { + /*ASSERT(0);*/ + goto FAIL; + } + +FAIL: + if (pAst != NULL) nodesDestroyNode(pAst); + return 0; +} + +int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + return 0; +} + +int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + SSdbRaw *pCommitRaw = mndStreamActionEncode(pStream); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + return 0; +} + int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) { SNode *pAst = NULL; @@ -356,24 +465,24 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { ASSERT(pTask->nodeId != 0); // vnode - if (pTask->nodeId > 0) { - SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pReq->head.vgId = htonl(pTask->nodeId); - pReq->taskId = pTask->taskId; - STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); - action.pCont = pReq; - action.contLen = sizeof(SVDropStreamTaskReq); - action.msgType = TDMT_VND_STREAM_TASK_DROP; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } + /*if (pTask->nodeId > 0) {*/ + SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + pReq->head.vgId = htonl(pTask->nodeId); + pReq->taskId = pTask->taskId; + STransAction action = {0}; + memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); + action.pCont = pReq; + action.contLen = sizeof(SVDropStreamTaskReq); + action.msgType = TDMT_VND_STREAM_TASK_DROP; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + /*}*/ return 0; } @@ -403,10 +512,9 @@ static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); streamObj.targetStbUid = mndGenerateUid(pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN); - streamObj.dbUid = pDb->uid; + streamObj.sourceDbUid = pDb->uid; streamObj.version = 1; streamObj.sql = pCreate->sql; - streamObj.createdBy = STREAM_CREATED_BY__USER; // TODO streamObj.fixedSinkVgId = 0; streamObj.smaId = 0; @@ -486,7 +594,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - // TODO check auth + // TODO check read auth for source and write auth for target +#if 0 pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -496,9 +605,52 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } +#endif - code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + // build stream obj from request + // + // schedule stream task for stream obj + // + SStreamObj streamObj = {0}; + if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) { + ASSERT(0); + mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); + goto _OVER; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); + if (pTrans == NULL) { + mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); + goto _OVER; + } + mndTransSetDbName(pTrans, streamObj.sourceDb); + // TODO + /*mndTransSetDbName(pTrans, streamObj.targetDb);*/ + mDebug("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); + + if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->conn.user) < 0) { + mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); + mndTransDrop(pTrans); + goto _OVER; + } + + if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { + mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); + mndTransDrop(pTrans); + goto _OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + goto _OVER; + } + + mndTransDrop(pTrans); + + /*code = mndCreateStream(pMnode, pReq, &createStreamReq, pDb);*/ + /*if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;*/ + code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { @@ -555,10 +707,54 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return code; } + // drop stream + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + sdbRelease(pMnode->pSdb, pStream); + return -1; + } + DROP_STREAM_OVER: return code; } +int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SStreamObj *pStream = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) break; + + if (pStream->sourceDbUid == pDb->uid || pStream->targetDbUid == pDb->uid) { + if (pStream->sourceDbUid != pStream->targetDbUid) { + sdbRelease(pSdb, pStream); + return -1; + } else { + // TODO drop all task on snode + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + sdbRelease(pSdb, pStream); + return -1; + } + } + } else { + sdbRelease(pSdb, pStream); + continue; + } + +#if 0 + if (mndSetDropOffsetStreamLogs(pMnode, pTrans, pStream) < 0) { + sdbRelease(pSdb, pStream); + goto END; + } +#endif + + sdbRelease(pSdb, pStream); + } + + return 0; +} + static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); @@ -574,7 +770,7 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; - if (pStream->dbUid == pDb->uid) { + if (pStream->sourceDbUid == pDb->uid) { numOfStreams++; } From 2f81a41f98ed0a49d7329eb561753971b6a4faba Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 16:16:40 +0800 Subject: [PATCH 36/90] refactor(sync): add syncIsReady --- include/libs/sync/sync.h | 1 + source/libs/sync/src/syncMain.c | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e694203563..fc9d419b1f 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -182,6 +182,7 @@ void syncStart(int64_t rid); void syncStop(int64_t rid); int32_t syncSetStandby(int64_t rid); ESyncState syncGetMyRole(int64_t rid); +bool syncIsReady(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); void syncGetEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 05a41daa9e..cc2c82fa5f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -381,6 +381,17 @@ ESyncState syncGetMyRole(int64_t rid) { return state; } +bool syncIsReady(int64_t rid) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return false; + } + assert(rid == pSyncNode->rid); + bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && pSyncNode->restoreFinish; + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return b; +} + bool syncIsRestoreFinish(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { From 87300b2ab047d2fecdffb0c4b542c8379b3af8d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 16:21:01 +0800 Subject: [PATCH 37/90] fix(query): set correct error code. --- source/libs/executor/src/executorimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9ce22431e1..bcd7b7f86f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4532,12 +4532,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); if (pDataReader == NULL && terrno != 0) { + pTaskInfo->code = terrno; return NULL; } int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { tsdbCleanupReadHandle(pDataReader); + pTaskInfo->code = terrno; return NULL; } @@ -4546,6 +4548,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo taosArrayDestroy(groupKeys); if (code) { tsdbCleanupReadHandle(pDataReader); + pTaskInfo->code = terrno; return NULL; } From 32d8e60ce249dc3cc571736bf2c7eba0327a2613 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 16:22:58 +0800 Subject: [PATCH 38/90] refactor(sma): create --- source/dnode/mnode/impl/inc/mndDef.h | 8 +-- source/dnode/mnode/impl/inc/mndScheduler.h | 2 + source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndDef.c | 79 ++++++++++++---------- source/dnode/mnode/impl/src/mndScheduler.c | 2 - source/dnode/mnode/impl/src/mndSma.c | 38 ++++++++++- source/dnode/mnode/impl/src/mndStream.c | 8 ++- 7 files changed, 93 insertions(+), 46 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index db18f87830..4daeeaa9bf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -546,13 +546,10 @@ typedef struct { int64_t createTime; int64_t updateTime; int32_t version; - // TODO remove it int64_t smaId; // 0 for unused // info int64_t uid; int8_t status; - // TODO remove it - int32_t vgNum; // config int8_t dropPolicy; int8_t trigger; @@ -565,9 +562,10 @@ typedef struct { char targetDb[TSDB_DB_FNAME_LEN]; char targetSTbName[TSDB_TABLE_FNAME_LEN]; int64_t targetStbUid; - // assigned when scheduling int32_t fixedSinkVgId; // 0 for shuffle - SVgObj fixedSinkVg; + // fixedSinkVg is not applicable for encode and decode + SVgObj fixedSinkVg; + // transformation char* sql; char* ast; diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index 80fe472c56..8e816d2dd6 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -32,6 +32,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, double filesFactor); +int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 65017027dc..6616e84488 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -31,7 +31,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); -int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans); +int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b9bd1e4298..20ba71992e 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -18,35 +18,35 @@ #include "mndConsumer.h" int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { - int32_t sz = 0; if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1; + + if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; + if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; + + if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; + + if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1; + if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; + + if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; + if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->targetDb) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->targetSTbName) < 0) return -1; if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->sourceDbUid) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->targetDbUid) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->version) < 0) return -1; - if (tEncodeI32(pEncoder, pObj->vgNum) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->status) < 0) return -1; - if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; - if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; - /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/ - if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; - // TODO encode tasks - if (pObj->tasks) { - sz = taosArrayGetSize(pObj->tasks); - } - if (tEncodeI32(pEncoder, sz) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; + if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + + int32_t sz = taosArrayGetSize(pObj->tasks); + if (tEncodeI32(pEncoder, sz) < 0) return -1; for (int32_t i = 0; i < sz; i++) { SArray *pArray = taosArrayGetP(pObj->tasks, i); int32_t innerSz = taosArrayGetSize(pArray); @@ -58,32 +58,38 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { } if (tEncodeSSchemaWrapper(pEncoder, &pObj->outputSchema) < 0) return -1; + return pEncoder->pos; } int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1; + + if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; + if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; + + if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; + + if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1; + if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; + + if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; + if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->targetDb) < 0) return -1; if (tDecodeCStrTo(pDecoder, pObj->targetSTbName) < 0) return -1; if (tDecodeI64(pDecoder, &pObj->targetStbUid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->sourceDbUid) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->targetDbUid) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1; - if (tDecodeI32(pDecoder, &pObj->vgNum) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1; - if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1; if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1; - if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1; + if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1; - /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/ + if (tDecodeCStrAlloc(pDecoder, &pObj->ast) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1; + pObj->tasks = NULL; int32_t sz; if (tDecodeI32(pDecoder, &sz) < 0) return -1; @@ -104,6 +110,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) { } if (tDecodeSSchemaWrapper(pDecoder, &pObj->outputSchema) < 0) return -1; + return 0; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index c53e97cd53..6f8fc748c2 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -346,8 +346,6 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - ASSERT(pStream->vgNum == 0); - int32_t totLevel = LIST_LENGTH(pPlan->pSubplans); ASSERT(totLevel <= 2); pStream->tasks = taosArrayInit(totLevel, sizeof(void*)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 54fd0fe867..3eb5b03efd 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -20,6 +20,7 @@ #include "mndDnode.h" #include "mndInfoSchema.h" #include "mndMnode.h" +#include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" #include "mndStream.h" @@ -563,6 +564,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.smaId = smaObj.uid; streamObj.watermark = 0; streamObj.trigger = STREAM_TRIGGER_AT_ONCE; + streamObj.ast = strdup(smaObj.ast); if (mndAllocSmaVgroup(pMnode, pDb, &streamObj.fixedSinkVg) != 0) { mError("sma:%s, failed to create since %s", smaObj.name, terrstr()); @@ -571,6 +573,39 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea smaObj.dstVgId = streamObj.fixedSinkVg.vgId; streamObj.fixedSinkVgId = smaObj.dstVgId; + SNode *pAst = NULL; + if (nodesStringToNode(streamObj.ast, &pAst) < 0) { + ASSERT(0); + return -1; + } + + // extract output schema from ast + if (qExtractResultSchema(pAst, (int32_t *)&streamObj.outputSchema.nCols, &streamObj.outputSchema.pSchema) != 0) { + ASSERT(0); + return -1; + } + + SQueryPlan *pPlan = NULL; + SPlanContext cxt = { + .pAstRoot = pAst, + .topicQuery = false, + .streamQuery = true, + .triggerType = streamObj.trigger, + .watermark = streamObj.watermark, + }; + + if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { + ASSERT(0); + return -1; + } + + // save physcial plan + if (nodesNodeToString((SNode *)pPlan, false, &streamObj.physicalPlan, NULL) != 0) { + ASSERT(0); + return -1; + } + if (pAst != NULL) nodesDestroyNode(pAst); + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; @@ -585,7 +620,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; // if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; - if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER; + if (mndScheduleStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; + if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index de0b25b1dc..c1df9dd68c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -640,6 +640,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + if (mndPersistStream(pMnode, pTrans, &streamObj) < 0) { + mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); + mndTransDrop(pTrans); + goto _OVER; + } + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); @@ -707,7 +713,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return code; } - // drop stream + // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); return -1; From 44e733645cf8fd75b5fc291e960d39b152405062 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 16:35:55 +0800 Subject: [PATCH 39/90] ci: remove unstable case --- source/dnode/mnode/impl/inc/mndStream.h | 3 +-- source/dnode/mnode/impl/src/mndStream.c | 28 +++++++++++++------------ tests/system-test/fulltest.sh | 2 +- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 6616e84488..69385c3a46 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -31,9 +31,8 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); -int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); - int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c1df9dd68c..0d5bc9b021 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -607,10 +607,18 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } #endif + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); + if (pTrans == NULL) { + mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); + goto _OVER; + } + + mndTransSetDbName(pTrans, createStreamReq.sourceDB); + // TODO + /*mndTransSetDbName(pTrans, streamObj.targetDb);*/ + mDebug("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); + // build stream obj from request - // - // schedule stream task for stream obj - // SStreamObj streamObj = {0}; if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) { ASSERT(0); @@ -618,34 +626,28 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); - if (pTrans == NULL) { - mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); - goto _OVER; - } - mndTransSetDbName(pTrans, streamObj.sourceDb); - // TODO - /*mndTransSetDbName(pTrans, streamObj.targetDb);*/ - mDebug("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); - + // create stb for stream if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->conn.user) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; } + // schedule stream task for stream obj if (mndScheduleStream(pMnode, pTrans, &streamObj) < 0) { mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; } + // add stream to trans if (mndPersistStream(pMnode, pTrans, &streamObj) < 0) { mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; } + // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 552ed26f34..b9f7e0f2eb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -101,7 +101,7 @@ python3 ./test.py -f 2-query/tail.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py +#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py From e224d404806726b1277735c8d182de1bf958b8fa Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 16 Jun 2022 16:58:55 +0800 Subject: [PATCH 40/90] feat: the unique function and tail function are rewritten as the corresponding clauses --- include/libs/function/functionMgt.h | 2 + include/libs/nodes/querynodes.h | 2 + include/util/taoserror.h | 3 + source/libs/function/inc/functionMgtInt.h | 2 + source/libs/function/src/builtins.c | 14 +- source/libs/function/src/functionMgt.c | 8 +- source/libs/parser/src/parTranslater.c | 177 ++++++++++++++++++++ source/libs/parser/src/parUtil.c | 10 +- source/libs/parser/test/parInitialCTest.cpp | 6 + source/libs/parser/test/parSelectTest.cpp | 34 ++++ source/libs/planner/src/planner.c | 2 +- source/libs/planner/test/planBasicTest.cpp | 18 ++ 12 files changed, 264 insertions(+), 14 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index dcc0a2ba4b..8e003eea7f 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -185,6 +185,8 @@ bool fmIsUserDefinedFunc(int32_t funcId); bool fmIsDistExecFunc(int32_t funcId); bool fmIsForbidFillFunc(int32_t funcId); bool fmIsForbidStreamFunc(int32_t funcId); +bool fmIsForbidWindowFunc(int32_t funcId); +bool fmIsForbidGroupByFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 989dcc9492..3adc619567 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -247,6 +247,8 @@ typedef struct SSelectStmt { bool hasIndefiniteRowsFunc; bool hasSelectFunc; bool hasSelectValFunc; + bool hasUniqueFunc; + bool hasTailFunc; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4f93ce9e4a..e6c3cf7141 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -658,6 +658,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG TAOS_DEF_ERROR_CODE(0, 0x2656) #define TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2657) #define TSDB_CODE_PAR_INVALID_WINDOW_PC TAOS_DEF_ERROR_CODE(0, 0x2658) +#define TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2659) +#define TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x265A) +#define TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x265B) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index 1443c26820..c02c09e18a 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -44,6 +44,8 @@ extern "C" { #define FUNC_MGT_FORBID_FILL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(15) #define FUNC_MGT_INTERVAL_INTERPO_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(16) #define FUNC_MGT_FORBID_STREAM_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(17) +#define FUNC_MGT_FORBID_WINDOW_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(18) +#define FUNC_MGT_FORBID_GROUP_BY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(19) #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b0741908a6..1cfa4d201e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1017,7 +1017,7 @@ static int32_t translateDerivative(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - SNode* pParamNode2 = nodesListGetNode(pFunc->pParameterList, 2); + SNode* pParamNode2 = nodesListGetNode(pFunc->pParameterList, 2); SValueNode* pValue2 = (SValueNode*)pParamNode2; pValue2->notReserved = true; @@ -1085,12 +1085,7 @@ static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (QUERY_NODE_COLUMN != nodeType(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE can only be columns"); - } - - pFunc->node.resType = ((SExprNode*)pPara)->resType; + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; return TSDB_CODE_SUCCESS; } @@ -2114,7 +2109,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tail", .type = FUNCTION_TYPE_TAIL, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -2124,7 +2119,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | + FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 0c762513cb..7dbe456d4a 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -165,6 +165,10 @@ bool fmIsIntervalInterpoFunc(int32_t funcId) { return isSpecificClassifyFunc(fun bool fmIsForbidStreamFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_STREAM_FUNC); } +bool fmIsForbidWindowFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_WINDOW_FUNC); } + +bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_GROUP_BY_FUNC); } + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { @@ -206,7 +210,7 @@ bool fmIsInvertible(int32_t funcId) { return res; } -//function has same input/output type +// function has same input/output type bool fmIsSameInOutType(int32_t funcId) { bool res = false; switch (funcMgtBuiltins[funcId].type) { @@ -301,7 +305,7 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio nodesDestroyList(pParameterList); return TSDB_CODE_OUT_OF_MEMORY; } - //overwrite function restype set by translate function + // overwrite function restype set by translate function if (fmIsSameInOutType(pSrcFunc->funcId)) { (*pMergeFunc)->node.resType = pSrcFunc->node.resType; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5d26a98dc8..f17d91f042 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1098,11 +1098,43 @@ static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctio return TSDB_CODE_SUCCESS; } +static int32_t translateForbidWindowFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsForbidWindowFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (NULL != pCxt->pCurrSelectStmt->pWindow) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC, pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateForbidStreamFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsForbidStreamFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (pCxt->createStream) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC, pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateForbidGroupByFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsForbidGroupByFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (NULL != pCxt->pCurrSelectStmt->pGroupByList) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC, pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) { if (NULL != pSelect) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); + pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); } } @@ -1130,6 +1162,15 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = translateWindowPseudoColumnFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = translateForbidWindowFunc(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = translateForbidStreamFunc(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = translateForbidGroupByFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == pCxt->errCode) { setFuncClassification(pCxt->pCurrSelectStmt, pFunc); } @@ -2116,6 +2157,136 @@ static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect return pCxt->errCode; } +typedef struct SRwriteUniqueCxt { + STranslateContext* pTranslateCxt; + SNode* pExpr; +} SRwriteUniqueCxt; + +static EDealRes rewriteSeletcValueFunc(STranslateContext* pCxt, SNode** pNode) { + SFunctionNode* pFirst = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFirst) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + strcpy(pFirst->functionName, "first"); + TSWAP(pFirst->pParameterList, ((SFunctionNode*)*pNode)->pParameterList); + nodesDestroyNode(*pNode); + *pNode = (SNode*)pFirst; + pCxt->errCode = fmGetFuncInfo(pFirst, pCxt->msgBuf.buf, pCxt->msgBuf.len); + pCxt->pCurrSelectStmt->hasAggFuncs = true; + return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; +} + +static EDealRes rewriteUniqueFunc(SNode** pNode, void* pContext) { + SRwriteUniqueCxt* pCxt = pContext; + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)*pNode; + if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) { + SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); + NODES_CLEAR_LIST(pFunc->pParameterList); + nodesDestroyNode(*pNode); + *pNode = pExpr; + pCxt->pExpr = pExpr; + return DEAL_RES_IGNORE_CHILD; + } else if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType) { + return rewriteSeletcValueFunc(pCxt->pTranslateCxt, pNode); + } + } + return DEAL_RES_CONTINUE; +} + +static SNode* createGroupingSet(SNode* pExpr) { + SGroupingSetNode* pGroupingSet = (SGroupingSetNode*)nodesMakeNode(QUERY_NODE_GROUPING_SET); + if (NULL == pGroupingSet) { + return NULL; + } + pGroupingSet->groupingSetType = GP_TYPE_NORMAL; + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pGroupingSet->pParameterList, nodesCloneNode(pExpr))) { + nodesDestroyNode((SNode*)pGroupingSet); + return NULL; + } + return (SNode*)pGroupingSet; +} + +static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (!pSelect->hasUniqueFunc) { + return TSDB_CODE_SUCCESS; + } + + SRwriteUniqueCxt cxt = {.pTranslateCxt = pCxt, .pExpr = NULL}; + nodesRewriteExprs(pSelect->pProjectionList, rewriteUniqueFunc, &cxt); + if (TSDB_CODE_SUCCESS == cxt.pTranslateCxt->errCode) { + cxt.pTranslateCxt->errCode = nodesListMakeStrictAppend(&pSelect->pGroupByList, createGroupingSet(cxt.pExpr)); + } + pSelect->hasIndefiniteRowsFunc = false; + return cxt.pTranslateCxt->errCode; +} + +typedef struct SRwriteTailCxt { + STranslateContext* pTranslateCxt; + int64_t limit; + int64_t offset; +} SRwriteTailCxt; + +static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) { + SRwriteTailCxt* pCxt = pContext; + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)*pNode; + if (FUNCTION_TYPE_TAIL == pFunc->funcType) { + pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i; + if (3 == LIST_LENGTH(pFunc->pParameterList)) { + pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i; + } + SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); + NODES_CLEAR_LIST(pFunc->pParameterList); + nodesDestroyNode(*pNode); + *pNode = pExpr; + return DEAL_RES_IGNORE_CHILD; + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) { + *pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + if (NULL == *pOutput) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pOutput)->limit = pCxt->limit; + (*pOutput)->offset = pCxt->offset; + return TSDB_CODE_SUCCESS; +} + +static SNode* createOrderByExpr(STranslateContext* pCxt) { + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOrder) { + return NULL; + } + pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr); + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + nodesDestroyNode((SNode*)pOrder); + return NULL; + } + pOrder->order = ORDER_DESC; + pOrder->nullOrder = NULL_ORDER_FIRST; + return (SNode*)pOrder; +} + +static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (!pSelect->hasTailFunc) { + return TSDB_CODE_SUCCESS; + } + + SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1}; + nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt); + int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt)); + if (TSDB_CODE_SUCCESS == code) { + code = createLimieNode(&cxt, &pSelect->pLimit); + } + pSelect->hasIndefiniteRowsFunc = false; + return code; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrSelectStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect->pFromTable); @@ -2147,6 +2318,12 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkLimit(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteUniqueStmt(pCxt, pSelect); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteTailStmt(pCxt, pSelect); + } if (TSDB_CODE_SUCCESS == code) { code = rewriteTimelineFunc(pCxt, pSelect); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 42e887bb9d..d561b8ac4b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -185,9 +185,15 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG: return "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes"; case TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC: - return "%s function not allowed in fill query"; + return "%s function does not supportted in fill query"; case TSDB_CODE_PAR_INVALID_WINDOW_PC: - return "_WSTARTTS, _WENDTS and _WDURATION can only be used in window queries"; + return "_WSTARTTS, _WENDTS and _WDURATION can only be used in window query"; + case TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC: + return "%s function does not supportted in time window query"; + case TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC: + return "%s function does not supportted in stream query"; + case TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC: + return "%s function does not supportted in group query"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 8b2c2aade7..0286dba363 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -528,6 +528,12 @@ TEST_F(ParserInitialCTest, createStream) { clearCreateStreamReq(); } +TEST_F(ParserInitialCTest, createStreamSemanticCheck) { + useDb("root", "test"); + + run("CREATE STREAM s1 AS SELECT PERCENTILE(c1, 30) FROM t1", TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC); +} + TEST_F(ParserInitialCTest, createTable) { useDb("root", "test"); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index c87520e262..97978a90f4 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -161,6 +161,40 @@ TEST_F(ParserSelectTest, useDefinedFunc) { run("SELECT udf2(c1) FROM t1 GROUP BY c2"); } +TEST_F(ParserSelectTest, uniqueFunc) { + useDb("root", "test"); + + run("SELECT UNIQUE(c1) FROM t1"); + + run("SELECT UNIQUE(c2 + 10) FROM t1 WHERE c1 > 10"); + + run("SELECT UNIQUE(c2 + 10), ts, c2 FROM t1 WHERE c1 > 10"); +} + +TEST_F(ParserSelectTest, uniqueFuncSemanticCheck) { + useDb("root", "test"); + + run("SELECT UNIQUE(c1) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + + run("SELECT UNIQUE(c1) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); +} + +TEST_F(ParserSelectTest, tailFunc) { + useDb("root", "test"); + + run("SELECT TAIL(c1, 10) FROM t1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10"); +} + +TEST_F(ParserSelectTest, tailFuncSemanticCheck) { + useDb("root", "test"); + + run("SELECT TAIL(c1, 10) FROM t1 INTERVAL(10S)", TSDB_CODE_PAR_WINDOW_NOT_ALLOWED_FUNC); + + run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); +} + TEST_F(ParserSelectTest, groupBy) { useDb("root", "test"); diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index c4385d576e..e75c8375fb 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -21,7 +21,7 @@ static void dumpQueryPlan(SQueryPlan* pPlan) { char* pStr = NULL; nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); - planDebugL("Query Plan: %s", pStr); + planDebugL("QID:0x%" PRIx64 " Query Plan: %s", pPlan->queryId, pStr); taosMemoryFree(pStr); } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 4b84079f7b..0304c6708d 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -53,3 +53,21 @@ TEST_F(PlanBasicTest, func) { run("SELECT TOP(c1, 60) FROM t1"); } + +TEST_F(PlanBasicTest, uniqueFunc) { + useDb("root", "test"); + + run("SELECT UNIQUE(c1) FROM t1"); + + run("SELECT UNIQUE(c2 + 10) FROM t1 WHERE c1 > 10"); + + run("SELECT UNIQUE(c2 + 10), ts, c2 FROM t1 WHERE c1 > 10"); +} + +TEST_F(PlanBasicTest, tailFunc) { + useDb("root", "test"); + + run("SELECT TAIL(c1, 10) FROM t1"); + + run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10"); +} From e0346c2d5abc87ee427ba00bb2f3b940be0a98fa Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 17:07:31 +0800 Subject: [PATCH 41/90] refactor(stream): change msg name --- include/common/tmsg.h | 4 ++-- source/dnode/mnode/impl/src/mndStream.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 01d99ac705..ab0203adf8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2259,11 +2259,11 @@ typedef struct { typedef struct { char name[TSDB_STREAM_FNAME_LEN]; int8_t igNotExists; -} SMDropStreamTaskReq; +} SMDropStreamReq; typedef struct { int8_t reserved; -} SMDropStreamTaskRsp; +} SMDropStreamRsp; typedef struct { SMsgHead head; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0d5bc9b021..3a42c694c8 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -679,7 +679,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { /*SDbObj *pDb = NULL;*/ /*SUserObj *pUser = NULL;*/ - SMDropStreamTaskReq dropReq = *(SMDropStreamTaskReq *)pReq->pCont; + SMDropStreamReq dropReq = *(SMDropStreamReq *)pReq->pCont; pStream = mndAcquireStream(pMnode, dropReq.name); From 07288e26b2ffeb04aadb943cc67f28b64c2113ce Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 17:07:31 +0800 Subject: [PATCH 42/90] refactor(stream): change msg name --- include/common/tmsg.h | 4 ++-- source/dnode/mnode/impl/src/mndStream.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 01d99ac705..ab0203adf8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2259,11 +2259,11 @@ typedef struct { typedef struct { char name[TSDB_STREAM_FNAME_LEN]; int8_t igNotExists; -} SMDropStreamTaskReq; +} SMDropStreamReq; typedef struct { int8_t reserved; -} SMDropStreamTaskRsp; +} SMDropStreamRsp; typedef struct { SMsgHead head; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0d5bc9b021..3a42c694c8 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -679,7 +679,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { /*SDbObj *pDb = NULL;*/ /*SUserObj *pUser = NULL;*/ - SMDropStreamTaskReq dropReq = *(SMDropStreamTaskReq *)pReq->pCont; + SMDropStreamReq dropReq = *(SMDropStreamReq *)pReq->pCont; pStream = mndAcquireStream(pMnode, dropReq.name); From 26ca03af4f484ed4a3db4fef4cfffd8586cffb8b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 17:24:42 +0800 Subject: [PATCH 43/90] fix: crash while mnode sync --- source/dnode/mnode/impl/src/mndSync.c | 11 ++++++++--- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- source/libs/sync/src/syncRaftLog.c | 5 +++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3a243e36d3..aec54f7c7b 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -60,6 +60,7 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode)); } tsem_post(&pMgmt->syncSem); + pMgmt->transId = 0; } else { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { @@ -123,6 +124,7 @@ void mndReConfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbM mError("trans:-1, failed to propose sync reconfig since %s", tstrerror(pMgmt->errCode)); } tsem_post(&pMgmt->syncSem); + pMgmt->transId = 0; } } @@ -266,13 +268,16 @@ void mndSyncStart(SMnode *pMnode) { */ } -void mndSyncStop(SMnode *pMnode) {} +void mndSyncStop(SMnode *pMnode) { + if (pMnode->syncMgmt.transId != 0) { + tsem_post(&pMnode->syncMgmt.syncSem); + } +} bool mndIsMaster(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; - ESyncState state = syncGetMyRole(pMgmt->sync); - if (state != TAOS_SYNC_STATE_LEADER) { + if (!syncIsReady(pMgmt->sync)) { terrno = TSDB_CODE_SYN_NOT_LEADER; return false; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3715866bb8..874210306f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -220,7 +220,7 @@ int vnodeCommit(SVnode *pVnode) { info.state.committed = pVnode->state.applied; snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); if (vnodeSaveInfo(dir, &info) < 0) { - ASSERT(0); + // ASSERT(0); return -1; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 31ac44fa81..08ecb8e553 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -407,6 +407,9 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { } int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { + +/* + SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; // assert(walCommit(pWal, index) == 0); @@ -419,6 +422,8 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } +*/ + return 0; } From 782e8fb518982f19227207d3cf499e2100acd710 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 17:39:31 +0800 Subject: [PATCH 44/90] fix: set stop falg before stop sync --- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 4 ++++ source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index b7124dfaa5..012e61d239 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -148,6 +148,10 @@ static int32_t mmStart(SMnodeMgmt *pMgmt) { static void mmStop(SMnodeMgmt *pMgmt) { dDebug("mnode-mgmt start to stop"); + taosThreadRwlockWrlock(&pMgmt->lock); + pMgmt->stopped = 1; + taosThreadRwlockUnlock(&pMgmt->lock); + mndStop(pMgmt->pMnode); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 7cd7da1aa9..493f4ab85f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -220,9 +220,6 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { } void mmStopWorker(SMnodeMgmt *pMgmt) { - taosThreadRwlockWrlock(&pMgmt->lock); - pMgmt->stopped = 1; - taosThreadRwlockUnlock(&pMgmt->lock); while (pMgmt->refCount > 0) taosMsleep(10); tSingleWorkerCleanup(&pMgmt->monitorWorker); From bd518f3f71557f86b9d5b5273603ca6d14e066a3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 17:39:33 +0800 Subject: [PATCH 45/90] fix(query): remove group info during sort. --- source/libs/executor/inc/executorimpl.h | 8 --- source/libs/executor/inc/tsort.h | 6 --- source/libs/executor/src/executorimpl.c | 31 +++-------- source/libs/executor/src/scanoperator.c | 14 +---- source/libs/executor/src/sortoperator.c | 70 +++++-------------------- source/libs/executor/src/tsort.c | 57 ++++++-------------- 6 files changed, 37 insertions(+), 149 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b435446513..034e2893df 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -696,10 +696,6 @@ typedef struct SSortedMergeOperatorInfo { int32_t numOfResPerPage; char** groupVal; SArray *groupInfo; - - bool hasGroupId; - uint64_t groupId; - STupleHandle* prefetchedTuple; } SSortedMergeOperatorInfo; typedef struct SSortOperatorInfo { @@ -712,10 +708,6 @@ typedef struct SSortOperatorInfo { int64_t startTs; // sort start time uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - - STupleHandle *prefetchedTuple; - bool hasGroupId; - uint64_t groupId; } SSortOperatorInfo; typedef struct STagFilterOperatorInfo { diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index e731c55a7d..86ee841cc2 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -130,12 +130,6 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId); */ void* tsortGetValue(STupleHandle* pVHandle, int32_t colId); -/** - * - * @param pVHandle - * @return - */ -uint64_t tsortGetGroupId(STupleHandle* pVHandle); /** * * @param pSortHandle diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index bcd7b7f86f..daee71b666 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3030,31 +3030,12 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = NULL; - if (pInfo->prefetchedTuple == NULL) { - pTupleHandle = tsortNextTuple(pHandle); - } else { - pTupleHandle = pInfo->prefetchedTuple; - pInfo->groupId = tsortGetGroupId(pTupleHandle); - pInfo->prefetchedTuple = NULL; - } - + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { break; } - uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); - if (!pInfo->hasGroupId) { - pInfo->groupId = tupleGroupId; - pInfo->hasGroupId = true; - appendOneRowToDataBlock(p, pTupleHandle); - } else if (pInfo->groupId == tupleGroupId) { - appendOneRowToDataBlock(p, pTupleHandle); - } else { - pInfo->prefetchedTuple = pTupleHandle; - break; - } - + appendOneRowToDataBlock(p, pTupleHandle); if (p->info.rows >= capacity) { break; } @@ -3073,7 +3054,6 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo pDataBlock->info.rows = p->info.rows; pDataBlock->info.capacity = p->info.rows; - pDataBlock->info.groupId = pInfo->groupId; } blockDataDestroy(p); @@ -3339,7 +3319,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { doSetOperatorCompleted(pOperator); } - size_t rows = blockDataGetNumOfRows(pInfo->pRes); // pInfo->pRes : NULL; + size_t rows = blockDataGetNumOfRows(pInfo->pRes); pOperator->resultInfo.totalRows += rows; return (rows == 0) ? NULL : pInfo->pRes; @@ -4920,7 +4900,10 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { } SArray* extractPartitionColInfo(SNodeList* pNodeList) { - if(!pNodeList) return NULL; + if(!pNodeList) { + return NULL; + } + size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); if (pList == NULL) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1ffb1529d7..ddebba1c9e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2175,25 +2175,13 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, int32_t capa pTupleHandle = tsortNextTuple(pHandle); } else { pTupleHandle = pInfo->prefetchedTuple; - pInfo->groupId = tsortGetGroupId(pTupleHandle); - pInfo->prefetchedTuple = NULL; } if (pTupleHandle == NULL) { break; } - uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); - if (!pInfo->hasGroupId) { - pInfo->groupId = tupleGroupId; - pInfo->hasGroupId = true; - appendOneRowToDataBlock(p, pTupleHandle); - } else if (pInfo->groupId == tupleGroupId) { - appendOneRowToDataBlock(p, pTupleHandle); - } else { - pInfo->prefetchedTuple = pTupleHandle; - break; - } + appendOneRowToDataBlock(p, pTupleHandle); if (p->info.rows >= capacity) { break; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 2f2080d5fe..81899b68cd 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -40,15 +40,13 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = pSortInfo; - pInfo->pColMatchInfo = pColMatchColInfo; - pInfo->hasGroupId = false; - pInfo->prefetchedTuple = NULL; - pOperator->name = "SortOperator"; + pInfo->pSortInfo = pSortInfo; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; // lazy evaluation for the following parameter since the input datablock is not known till now. // pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + @@ -97,31 +95,12 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = NULL; - if (pInfo->prefetchedTuple == NULL) { - pTupleHandle = tsortNextTuple(pHandle); - } else { - pTupleHandle = pInfo->prefetchedTuple; - pInfo->groupId = tsortGetGroupId(pTupleHandle); - pInfo->prefetchedTuple = NULL; - } - + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { break; } - uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); - if (!pInfo->hasGroupId) { - pInfo->groupId = tupleGroupId; - pInfo->hasGroupId = true; - appendOneRowToDataBlock(p, pTupleHandle); - } else if (pInfo->groupId == tupleGroupId) { - appendOneRowToDataBlock(p, pTupleHandle); - } else { - pInfo->prefetchedTuple = pTupleHandle; - break; - } - + appendOneRowToDataBlock(p, pTupleHandle); if (p->info.rows >= capacity) { break; } @@ -140,7 +119,6 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i pDataBlock->info.rows = p->info.rows; pDataBlock->info.capacity = p->info.rows; - pDataBlock->info.groupId = pInfo->groupId; } blockDataDestroy(p); @@ -255,10 +233,7 @@ typedef struct SMultiwaySortMergeOperatorInfo { SSDataBlock* pInputBlock; int64_t startTs; // sort start time - - bool hasGroupId; - uint64_t groupId; - STupleHandle* prefetchedTuple; + uint64_t groupId; } SMultiwaySortMergeOperatorInfo; int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { @@ -312,31 +287,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = NULL; - if (pInfo->prefetchedTuple == NULL) { - pTupleHandle = tsortNextTuple(pHandle); - } else { - pTupleHandle = pInfo->prefetchedTuple; - pInfo->groupId = tsortGetGroupId(pTupleHandle); - pInfo->prefetchedTuple = NULL; - } - + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { break; } - uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); - if (!pInfo->hasGroupId) { - pInfo->groupId = tupleGroupId; - pInfo->hasGroupId = true; - appendOneRowToDataBlock(p, pTupleHandle); - } else if (pInfo->groupId == tupleGroupId) { - appendOneRowToDataBlock(p, pTupleHandle); - } else { - pInfo->prefetchedTuple = pTupleHandle; - break; - } - + appendOneRowToDataBlock(p, pTupleHandle); if (p->info.rows >= capacity) { break; } @@ -432,14 +388,12 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pInfo->hasGroupId = false; - pInfo->prefetchedTuple = NULL; pOperator->pTaskInfo = pTaskInfo; pInfo->bufPageSize = getProperSortPageSize(rowSize); uint32_t numOfSources = taosArrayGetSize(pSortInfo); - numOfSources = TMAX(2, numOfSources); + numOfSources = TMAX(4, numOfSources); pInfo->sortBufSize = numOfSources * pInfo->bufPageSize; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 5bcd58f8db..1502387360 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -557,59 +557,40 @@ static int32_t createInitialSources(SSortHandle* pHandle) { SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0); taosArrayClear(pHandle->pOrderedSource); - bool hasGroupId = false; - SSDataBlock* prefetchedDataBlock = NULL; - while (1) { - SSDataBlock* pBlock = NULL; - if (prefetchedDataBlock == NULL) { - pBlock = pHandle->fetchfp(source->param); - } else { - pBlock = prefetchedDataBlock; - prefetchedDataBlock = NULL; - } - + SSDataBlock* pBlock = pHandle->fetchfp(source->param); if (pBlock == NULL) { break; } - if (!hasGroupId) { - // calculate the buffer pages according to the total available buffers. + if (pHandle->pDataBlock == NULL) { pHandle->pageSize = getProperSortPageSize(blockDataGetRowSize(pBlock)); // todo, number of pages are set according to the total available sort buffer pHandle->numOfPages = 1024; sortBufSize = pHandle->numOfPages * pHandle->pageSize; - - hasGroupId = true; pHandle->pDataBlock = createOneDataBlock(pBlock, false); } - if (pHandle->pDataBlock->info.groupId == pBlock->info.groupId) { - // perform the scalar function calculation before apply the sort - if (pHandle->beforeFp != NULL) { - pHandle->beforeFp(pBlock, pHandle->param); - } + if (pHandle->beforeFp != NULL) { + pHandle->beforeFp(pBlock, pHandle->param); + } - int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); - if (code != 0) { - return code; - } + int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); + if (code != 0) { + return code; + } - size_t size = blockDataGetSize(pHandle->pDataBlock); - if (size > sortBufSize) { - // Perform the in-memory sort and then flush data in the buffer into disk. - int64_t p = taosGetTimestampUs(); - blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); + size_t size = blockDataGetSize(pHandle->pDataBlock); + if (size > sortBufSize) { + // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo); - int64_t el = taosGetTimestampUs() - p; - pHandle->sortElapsed += el; + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; - doAddToBuf(pHandle->pDataBlock, pHandle); - } - } else { - prefetchedDataBlock = pBlock; - pHandle->pDataBlock = createOneDataBlock(pBlock, false); + doAddToBuf(pHandle->pDataBlock, pHandle); } } @@ -758,10 +739,6 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { } } -uint64_t tsortGetGroupId(STupleHandle* pVHandle) { - return pVHandle->pBlock->info.groupId; -} - SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { SSortExecInfo info = {0}; From 15ac89590c215eb9568b3141a6c80f1a699bb3a6 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 17:41:01 +0800 Subject: [PATCH 46/90] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntries.c | 8 +- source/libs/sync/src/syncAppendEntriesReply.c | 16 ++- source/libs/sync/src/syncCommit.c | 6 +- source/libs/sync/src/syncMain.c | 72 ++++++----- source/libs/sync/src/syncRaftLog.c | 41 +++--- source/libs/sync/src/syncRespMgr.c | 21 +-- source/libs/sync/src/syncSnapshot.c | 120 ++++++++++-------- source/libs/sync/src/syncUtil.c | 20 ++- 8 files changed, 174 insertions(+), 130 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 89b4761212..bea51b2dd3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -713,8 +713,8 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - sDebug("vgId:%d sync event currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, ths->pRaftStore->currentTerm, - delBegin, delEnd); + sDebug("vgId:%d sync event %s currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, + syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, delBegin, delEnd); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); return code; @@ -995,8 +995,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs SyncIndex commitEnd = snapshot.lastApplyIndex; ths->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event currentTerm:%lu commit by snapshot from index:%ld to index:%ld, %s", ths->vgId, - ths->pRaftStore->currentTerm, commitBegin, commitEnd, syncUtilState2String(ths->state)); + sDebug("vgId:%d sync event %s currentTerm:%lu commit by snapshot from index:%ld to index:%ld", ths->vgId, + syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, commitBegin, commitEnd); } SyncIndex beginIndex = ths->commitIndex + 1; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 5290d7d28e..94fdcc2772 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -190,19 +190,23 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (gRaftDetailLog) { char* s = snapshotSender2Str(pSender); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d start sender first time, " + "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu " "sender:%s", - ths->vgId, ths->pRaftStore->currentTerm, host, port, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, s); + ths->vgId, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, host, port, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm, s); taosMemoryFree(s); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d start sender first time, lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d start sender first time, " + "lastApplyIndex:%ld " "lastApplyTerm:%lu lastConfigIndex:%ld privateTerm:%lu", - ths->vgId, ths->pRaftStore->currentTerm, host, port, pSender->snapshot.lastApplyIndex, - pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); + ths->vgId, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, host, port, + pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, + pSender->privateTerm); } } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index efdb5b24ce..e3431307fd 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -56,9 +56,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex commitEnd = snapshot.lastApplyIndex; pSyncNode->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event currentTerm:%lu commit by snapshot from index:%ld to index:%ld, %s", pSyncNode->vgId, - pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, snapshot.lastApplyIndex, - syncUtilState2String(pSyncNode->state)); + sDebug("vgId:%d sync event %s currentTerm:%lu commit by snapshot from index:%ld to index:%ld", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, + snapshot.lastApplyIndex); } // update commit index diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index cc2c82fa5f..57481ee3a5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -573,8 +573,9 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return -1; } assert(rid == pSyncNode->rid); - sDebug("vgId:%d sync event currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, - pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), pMsg->msgType); + sDebug("vgId:%d sync event %s currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), + pMsg->msgType); ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -583,8 +584,9 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; - sDebug("vgId:%d sync event currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, - pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), pMsg->msgType); + sDebug("vgId:%d sync event %s currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), + pMsg->msgType); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -830,7 +832,8 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // snapshot meta // pSyncNode->sMeta.lastConfigIndex = -1; - sDebug("vgId:%d sync event currentTerm:%lu sync open", pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu sync open", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pSyncNode->pRaftStore->currentTerm); return pSyncNode; } @@ -877,7 +880,8 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodeClose(SSyncNode* pSyncNode) { - sDebug("vgId:%d sync event currentTerm:%lu sync close", pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu sync close", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pSyncNode->pRaftStore->currentTerm); int32_t ret; assert(pSyncNode != NULL); @@ -1316,8 +1320,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { oldSenders[i] = (pSyncNode->senders)[i]; - sDebug("vgId:%d sync event currentTerm:%lu save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, - pSyncNode->pRaftStore->currentTerm, i, oldSenders[i], oldSenders[i]->privateTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, i, oldSenders[i], + oldSenders[i]->privateTerm); if (gRaftDetailLog) { ; } @@ -1369,9 +1374,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, privateTerm:%lu", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, (pSyncNode->replicasId)[i].addr, i, host, port, - oldSenders[j], oldSenders[j]->privateTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, privateTerm:%lu", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j], oldSenders[j]->privateTerm); (pSyncNode->senders)[i] = oldSenders[j]; oldSenders[j] = NULL; reset = true; @@ -1379,9 +1384,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex // reset replicaIndex int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; (pSyncNode->senders)[i]->replicaIndex = i; - sDebug("vgId:%d sync event currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, oldreplicaIndex, i, host, port, - (pSyncNode->senders)[i], reset); + sDebug("vgId:%d sync event %s currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); } } } @@ -1390,9 +1395,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sDebug("vgId:%d sync event currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, (pSyncNode->senders)[i], i, - (pSyncNode->senders)[i]->privateTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + (pSyncNode->senders)[i], i, (pSyncNode->senders)[i]->privateTerm); } } @@ -1400,8 +1405,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { snapshotSenderDestroy(oldSenders[i]); - sDebug("vgId:%d sync event currentTerm:%lu delete old sender %p replicaIndex:%d", pSyncNode->vgId, - pSyncNode->pRaftStore->currentTerm, oldSenders[i], i); + sDebug("vgId:%d sync event %s currentTerm:%lu delete old sender %p replicaIndex:%d", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, oldSenders[i], i); oldSenders[i] = NULL; } } @@ -1472,10 +1477,10 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { sDebug( - "vgId:%d sync event currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " + "vgId:%d sync event %s currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " "restoreFinish:%d, %s", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, - pSyncNode->restoreFinish, debugStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->restoreFinish, debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1512,9 +1517,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // reset restoreFinish pSyncNode->restoreFinish = false; - sDebug("vgId:%d sync event currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, restoreFinish:%d, %s", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, - pSyncNode->restoreFinish, debugStr); + sDebug("vgId:%d sync event %s currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, restoreFinish:%d, %s", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->restoreFinish, debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -2088,12 +2093,13 @@ const char* syncStr(ESyncState state) { static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - sDebug("vgId:%d sync event currentTerm:%lu begin leader transfer", ths->vgId, ths->pRaftStore->currentTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu begin leader transfer", ths->vgId, syncUtilState2String(ths->state), + ths->pRaftStore->currentTerm); if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { - sDebug("vgId:%d sync event currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, - ths->pRaftStore->currentTerm, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, + sDebug("vgId:%d sync event %s currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, + syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); // reset elect timer now! @@ -2216,8 +2222,9 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; - sDebug("vgId:%d sync event currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, - ths->pRaftStore->currentTerm, beginIndex, endIndex, syncUtilState2String(state)); + sDebug("vgId:%d sync event %s currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", + ths->vgId, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, beginIndex, endIndex, + syncUtilState2String(state)); // execute fsm if (ths->pFsm != NULL) { @@ -2265,8 +2272,9 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sDebug("vgId:%d sync event currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, - ths->pRaftStore->currentTerm, syncUtilState2String(ths->state), pEntry->index); + sDebug("vgId:%d sync event %s currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, + syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, syncUtilState2String(ths->state), + pEntry->index); } } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 31ac44fa81..308edc0b93 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -163,9 +163,9 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); - sDebug("vgId:%d sync event currentTerm:%lu write index:%ld, %s, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", - pData->pSyncNode->vgId, pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, - syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftCfg->isStandBy, + sDebug("vgId:%d sync event %s currentTerm:%lu write index:%ld, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", + pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), + pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); return code; @@ -323,10 +323,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { walFsync(pWal, true); sDebug( - "vgId:%d sync event currentTerm:%lu old write index:%ld, %s, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", - pData->pSyncNode->vgId, pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, - syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), - pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + "vgId:%d sync event %s currentTerm:%lu old write index:%ld, isStandBy:%d, msgType:%s,%d, " + "originalRpcType:%s,%d", + pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftStore->currentTerm, + pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), pEntry->msgType, + TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); return code; } @@ -407,18 +408,20 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { } int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { - SSyncLogStoreData* pData = pLogStore->data; - SWal* pWal = pData->pWal; - // assert(walCommit(pWal, index) == 0); - int32_t code = walCommit(pWal, index); - if (code != 0) { - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); - ASSERT(0); - } + /* + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + // assert(walCommit(pWal, index) == 0); + int32_t code = walCommit(pWal, index); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, + linuxErrMsg); ASSERT(0); + } + */ return 0; } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 5b127793d6..cc4004a6bd 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -46,9 +46,10 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, - pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, - pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); + sDebug("vgId:%d sync event %s currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, + pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return keyCode; @@ -71,9 +72,10 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p ahandle:%p", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, - index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); + sDebug("vgId:%d sync event %s currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, + pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return 1; // get one object @@ -90,9 +92,10 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p ahandle:%p", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, - index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); + sDebug("vgId:%d sync event %s currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, + pStub->rpcMsg.info.ahandle); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); taosThreadMutexUnlock(&(pObj->mutex)); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index d3785060d4..219a1a5a7d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -141,20 +141,22 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } @@ -283,29 +285,32 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } @@ -339,14 +344,16 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu send msg:%s", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, - pSender->privateTerm, msgStr); + "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu send " + "msg:%s", + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->privateTerm, + msgStr); taosMemoryFree(msgStr); } else { - sDebug("vgId:%d sync event currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu", - pSender->pSyncNode->vgId, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, - pSender->ack, pSender->privateTerm); + sDebug("vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu", + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -579,17 +586,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, - pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld privateTerm:%lu", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, - pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -626,18 +635,18 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { bool isDrop; if (IamInNew) { sDebug( - "vgId:%d sync event currentTerm:%lu update config by snapshot, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s currentTerm:%lu update config by snapshot, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld ", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); syncNodeUpdateConfig(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex, &isDrop); } else { sDebug( - "vgId:%d sync event currentTerm:%lu do not update config by snapshot, I am not in newCfg, " + "vgId:%d sync event %s currentTerm:%lu do not update config by snapshot, I am not in newCfg, " "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld ", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, pMsg->lastIndex, pMsg->lastTerm, - pMsg->lastConfigIndex); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); } // change isStandBy to normal @@ -662,20 +671,21 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu, raft log:%s", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pMsg->lastIndex + 1, - snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, pReceiver->privateTerm, - logSimpleStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, + pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, + pReceiver->privateTerm, logSimpleStr); taosMemoryFree(logSimpleStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pMsg->lastIndex + 1, - snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, pReceiver->privateTerm); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, + pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, + pReceiver->privateTerm); } pReceiver->pWriter = NULL; @@ -686,17 +696,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { @@ -711,20 +723,22 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv " "msg:%s", - pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pReceiver->pSyncNode->vgId, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, - pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, + pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { @@ -744,19 +758,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, - pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pSyncNode->vgId, pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, - pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, + pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } } else { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4026596548..cbc1298113 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -168,14 +168,26 @@ char* syncUtilRaftId2Str(const SRaftId* p) { } const char* syncUtilState2String(ESyncState state) { + /* + if (state == TAOS_SYNC_STATE_FOLLOWER) { + return "TAOS_SYNC_STATE_FOLLOWER"; + } else if (state == TAOS_SYNC_STATE_CANDIDATE) { + return "TAOS_SYNC_STATE_CANDIDATE"; + } else if (state == TAOS_SYNC_STATE_LEADER) { + return "TAOS_SYNC_STATE_LEADER"; + } else { + return "TAOS_SYNC_STATE_UNKNOWN"; + } + */ + if (state == TAOS_SYNC_STATE_FOLLOWER) { - return "TAOS_SYNC_STATE_FOLLOWER"; + return "follower"; } else if (state == TAOS_SYNC_STATE_CANDIDATE) { - return "TAOS_SYNC_STATE_CANDIDATE"; + return "candidate"; } else if (state == TAOS_SYNC_STATE_LEADER) { - return "TAOS_SYNC_STATE_LEADER"; + return "leader"; } else { - return "TAOS_SYNC_STATE_UNKNOWN"; + return "state_error"; } } From 46c09f2c84ace81257bf6e241540e15604e096f0 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 18:08:01 +0800 Subject: [PATCH 47/90] refactor(sync): add trace log --- source/libs/sync/src/syncMain.c | 24 ++++++++++++++++++++---- source/libs/sync/src/syncSnapshot.c | 15 ++++++++++++--- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 57481ee3a5..ffa6dc1291 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2178,8 +2178,16 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE // change isStandBy to normal if (!isDrop) { - char tmpbuf[128]; - snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d", oldSyncCfg.replicaNum, newSyncCfg.replicaNum); + char tmpbuf[512]; + char* oldStr = syncCfg2Str(&oldSyncCfg); + char* newStr = syncCfg2Str(&newSyncCfg); + syncUtilJson2Line(oldStr); + syncUtilJson2Line(newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, oldStr, newStr); + taosMemoryFree(oldStr); + taosMemoryFree(newStr); + if (ths->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(ths, tmpbuf); } else { @@ -2187,8 +2195,16 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE } } } else { - char tmpbuf[128]; - snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d", oldSyncCfg.replicaNum, newSyncCfg.replicaNum); + char tmpbuf[512]; + char* oldStr = syncCfg2Str(&oldSyncCfg); + char* newStr = syncCfg2Str(&newSyncCfg); + syncUtilJson2Line(oldStr); + syncUtilJson2Line(newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, oldStr, newStr); + taosMemoryFree(oldStr); + taosMemoryFree(newStr); + syncNodeBecomeFollower(ths, tmpbuf); } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 219a1a5a7d..68336499bf 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -611,7 +611,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { - int32_t oldReplicaNum = pSyncNode->replicaNum; + // int32_t oldReplicaNum = pSyncNode->replicaNum; + SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg; // update new config myIndex SSyncCfg newSyncCfg = pMsg->lastConfig; @@ -651,8 +652,16 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // change isStandBy to normal if (!isDrop) { - char tmpbuf[128]; - snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d", oldReplicaNum, newSyncCfg.replicaNum); + char tmpbuf[512]; + char *oldStr = syncCfg2Str(&oldSyncCfg); + char *newStr = syncCfg2Str(&newSyncCfg); + syncUtilJson2Line(oldStr); + syncUtilJson2Line(newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, oldStr, newStr); + taosMemoryFree(oldStr); + taosMemoryFree(newStr); + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(pSyncNode, tmpbuf); } else { From 7b98430906e03208282be22ac084585172ac0a01 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 16 Jun 2022 17:26:52 +0800 Subject: [PATCH 48/90] feat(stream): stream partition by --- source/libs/executor/src/timewindowoperator.c | 170 ++++++++++-------- .../tsim/stream/distributeInterval0.sim | 37 +++- tests/script/tsim/stream/partitionby.sim | 62 ++++++- 3 files changed, 195 insertions(+), 74 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f8972a02a1..d7ae823522 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1257,6 +1257,10 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { return TSDB_CODE_SUCCESS; } +bool isCloseWindow(STimeWindow *pWin, STimeWindowAggSupp* pSup) { + return pWin->ekey < pSup->maxTs - pSup->waterMark; +} + static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, SArray* closeWins) { void* pIte = NULL; @@ -1269,7 +1273,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); - if (win.ekey < pSup->maxTs - pSup->waterMark) { + if (isCloseWindow(&win, pSup)) { char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); taosHashRemove(pHashMap, keyBuf, keyLen); @@ -2036,59 +2040,6 @@ _error: return NULL; } -static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId, - SArray* pUpdated) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfExprs; - int32_t step = 1; - bool ascScan = true; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardRows = 0; - - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } else { - return; - } - - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, NULL); - while (1) { - int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { - saveResultRow(pResult, tableGroupId, pUpdated); - } - // window start(end) key interpolation - // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, - // forwardRows); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - int32_t prevEndPos = (forwardRows - 1) * step + startPos; - ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); - if (startPos < 0) { - break; - } - } -} - bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; } void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, @@ -2130,6 +2081,74 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArra } } +bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { + SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, + pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); + return p1 == NULL; +} + +static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SArray* pUpdated) { + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; + + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } else { + return; + } + + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, NULL); + while (1) { + if (isFinalInterval(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && + isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) { + SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); + taosArrayPush(pUpWins, &nextWin); + rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, + pOperatorInfo->numOfExprs, pOperatorInfo->pTaskInfo); + taosArrayDestroy(pUpWins); + } + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pos->key = pResult->win.skey; + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { + saveResultRow(pResult, tableGroupId, pUpdated); + } + // window start(end) key interpolation + // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + int32_t prevEndPos = (forwardRows - 1) * step + startPos; + ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); + if (startPos < 0) { + break; + } + } +} + static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) { taosHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); @@ -2169,6 +2188,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + TSKEY maxTs = INT64_MIN; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2222,6 +2242,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); + doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); if (isFinalInterval(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -2238,10 +2259,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pChildOp, pChInfo->binfo.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); } - doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.window.ekey); } + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (isFinalInterval(pInfo)) { closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); } @@ -2564,7 +2585,7 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t } static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, - int32_t groupId, int32_t numOfOutput, int32_t* rowCellInfoOffset, + uint64_t groupId, int32_t numOfOutput, int32_t* rowCellInfoOffset, SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { assert(pWinInfo->win.skey <= pWinInfo->win.ekey); // too many time window in query @@ -2642,7 +2663,7 @@ int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) return size - startIndex - 1; } -void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, int32_t groupId, +void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) { SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex); SResultRow* pCurResult = NULL; @@ -2667,13 +2688,18 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, } } +typedef struct SWinRes { + TSKEY ts; + uint64_t groupId; +} SWinRes; + static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, SHashObj* pStDeleted) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; bool masterScan = true; int32_t numOfOutput = pOperator->numOfExprs; - int64_t groupId = pSDataBlock->info.groupId; + uint64_t groupId = pSDataBlock->info.groupId; int64_t gap = pInfo->gap; int64_t code = TSDB_CODE_SUCCESS; @@ -2693,7 +2719,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; for (int32_t i = 0; i < pSDataBlock->info.rows;) { int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, gap, &winIndex); + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], groupId, gap, &winIndex); winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { @@ -2709,7 +2735,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } pCurWin->isClosed = false; if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &(pCurWin->win.skey), sizeof(TSKEY)); + SWinRes value = {.ts = pCurWin->win.skey, .groupId = groupId}; + code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes)); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -2736,7 +2763,7 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* } } -static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated, int32_t groupId) { +static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) { void* pData = NULL; size_t keyLen = 0; while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) { @@ -2746,9 +2773,9 @@ static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated, int32_t if (pos == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pos->groupId = groupId; + pos->groupId = ((SWinRes*)pData)->groupId; pos->pos = *(SResultRowPosition*)key; - *(int64_t*)pos->key = *(uint64_t*)pData; + *(int64_t*)pos->key = ((SWinRes*)pData)->ts; taosArrayPush(pUpdated, &pos); } return TSDB_CODE_SUCCESS; @@ -2815,7 +2842,9 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra __get_win_info_ fn) { // Todo(liuyao) save window to tdb void **pIte = NULL; + size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen); SArray *pWins = (SArray *) (*pIte); int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { @@ -2825,7 +2854,7 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra if (!pSeWin->isClosed) { pSeWin->isClosed = true; if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); + int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed); pSeWin->isOutput = true; } } @@ -2892,7 +2921,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, pChildOp->numOfExprs, pChildInfo->gap, NULL); - rebuildTimeWindow(pInfo, pWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, pOperator->pTaskInfo); + rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->numOfExprs, pOperator->pTaskInfo); } taosArrayDestroy(pWins); continue; @@ -2916,7 +2945,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getSessionWinInfo); - copyUpdateResult(pStUpdated, pUpdated, pBInfo->pRes->info.groupId); + copyUpdateResult(pStUpdated, pUpdated); taosHashCleanup(pStUpdated); finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, @@ -3216,8 +3245,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } pCurWin->winInfo.isClosed = false; if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &(pCurWin->winInfo.win.skey), - sizeof(TSKEY)); + SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId}; + code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), + &value, sizeof(SWinRes)); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -3274,7 +3304,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getStateWinInfo); - copyUpdateResult(pSeUpdated, pUpdated, pBInfo->pRes->info.groupId); + copyUpdateResult(pSeUpdated, pUpdated); taosHashCleanup(pSeUpdated); finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index f4f3e04f0a..b720272116 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -173,4 +173,39 @@ endi sql select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file + +sql create database test1 vgroups 1; +sql use test1; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create stream stream_t2 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); +sql insert into ts1 values(1648791222001,2,2,3); +sql insert into ts2 values(1648791211000,1,2,3); +sql insert into ts2 values(1648791222001,2,2,3); + +$loop_count = 0 +loop2: +sql select * from streamtST1; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 2 then + print =====data01=$data01 + goto loop2 +endi + +#rows 1 +if $data11 != 2 then + print =====data11=$data11 + goto loop2 +endi + +system sh/stop_dnodes.sh \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index df1e096551..b84a01eb4a 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -34,6 +34,7 @@ print =====rows=$rows goto loop0 endi +print =====loop0 sql create database test1 vgroups 1; sql use test1; @@ -51,7 +52,7 @@ sql insert into ts2 values(1648791211000,1,2,3); $loop_count = 0 -loop0: +loop1: sleep 300 sql select * from streamt; @@ -62,7 +63,62 @@ endi if $rows != 2 then print =====rows=$rows -goto loop0 +goto loop1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +print =====loop1 + +sql create database test2 vgroups 1; +sql use test2; +sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); + +sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; +sql insert into ts1 values(1648791211000,1,2,3,1); +sql insert into ts1 values(1648791222001,2,2,3,2); +sql insert into ts2 values(1648791211000,1,2,3,3); +sql insert into ts2 values(1648791222001,2,2,3,4); + +sql insert into ts2 values(1648791222002,2,2,3,5); +sql insert into ts2 values(1648791222002,2,2,3,6); + +sql insert into ts1 values(1648791211000,1,2,3,1); +sql insert into ts1 values(1648791222001,2,2,3,2); +sql insert into ts2 values(1648791211000,1,2,3,3); +sql insert into ts2 values(1648791222001,2,2,3,4); + +$loop_count = 0 + +loop2: +sleep 300 +sql select * from streamtST; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then +print =====data01=$data01 +goto loop2 +endi + +if $data02 != 1 then +print =====data02=$data02 +goto loop2 +endi + +if $data03 != 1 then +print =====data03=$data03 +goto loop2 +endi + +if $data04 != 2 then +print =====data04=$data04 +goto loop2 +endi + +print =====loop2 + +system sh/stop_dnodes.sh \ No newline at end of file From ab47de1dae88097fc546a4e57f5dbe81a4e11b80 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 18:30:22 +0800 Subject: [PATCH 49/90] fix(query): adjust group limitation. --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index fb79c53384..66849e49b8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3508,7 +3508,7 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); - if (pProjectInfo->slimit.limit == -1 || pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { + if (pProjectInfo->slimit.limit != -1 && pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { pOperator->status = OP_EXEC_DONE; } From c1de4df95bde0ec91dd59203d6403042fad52f20 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 16 Jun 2022 18:33:45 +0800 Subject: [PATCH 50/90] fix: fix page capacity calculation --- source/libs/tdb/src/db/tdbPage.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 92272fb438..78470b6256 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -246,14 +246,17 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { int tdbPageCapacity(int pageSize, int amHdrSize) { int szPageHdr; + int minCellIndexSize; // at least one cell in cell index if (pageSize < 65536) { szPageHdr = pageMethods.szPageHdr; + minCellIndexSize = pageMethods.szOffset; } else { szPageHdr = pageLargeMethods.szPageHdr; + minCellIndexSize = pageLargeMethods.szOffset; } - return pageSize - szPageHdr - amHdrSize; + return pageSize - szPageHdr - amHdrSize - sizeof(SPageFtr) - minCellIndexSize; } static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { @@ -599,4 +602,4 @@ SPageMethods pageLargeMethods = { setLPageCellOffset, // setCellOffset getLPageFreeCellInfo, // getFreeCellInfo setLPageFreeCellInfo // setFreeCellInfo -}; \ No newline at end of file +}; From 3fdd0c30ac31e236aece2e7a9743001b9db8a716 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 19:08:44 +0800 Subject: [PATCH 51/90] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntries.c | 10 +- source/libs/sync/src/syncAppendEntriesReply.c | 8 +- source/libs/sync/src/syncCommit.c | 6 +- source/libs/sync/src/syncMain.c | 95 ++++++++------ source/libs/sync/src/syncRaftLog.c | 18 +-- source/libs/sync/src/syncRespMgr.c | 29 +++-- source/libs/sync/src/syncSnapshot.c | 122 +++++++++++------- 7 files changed, 167 insertions(+), 121 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index bea51b2dd3..1235e8eda6 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -713,8 +713,8 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - sDebug("vgId:%d sync event %s currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, - syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, delBegin, delEnd); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, + syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, delBegin, delEnd); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); return code; @@ -995,8 +995,10 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs SyncIndex commitEnd = snapshot.lastApplyIndex; ths->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event %s currentTerm:%lu commit by snapshot from index:%ld to index:%ld", ths->vgId, - syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, commitBegin, commitEnd); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu commit by snapshot from index:%ld to index:%ld", + ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, + commitBegin, commitEnd); } SyncIndex beginIndex = ths->commitIndex + 1; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 94fdcc2772..f934f9a268 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -190,21 +190,21 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (gRaftDetailLog) { char* s = snapshotSender2Str(pSender); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d start sender first time, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d start sender first time, " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu " "sender:%s", - ths->vgId, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, host, port, + ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, s); taosMemoryFree(s); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d start sender first time, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d start sender first time, " "lastApplyIndex:%ld " "lastApplyTerm:%lu lastConfigIndex:%ld privateTerm:%lu", - ths->vgId, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, host, port, + ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, host, port, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index e3431307fd..d010728c78 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -56,9 +56,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex commitEnd = snapshot.lastApplyIndex; pSyncNode->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event %s currentTerm:%lu commit by snapshot from index:%ld to index:%ld", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, - snapshot.lastApplyIndex); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu commit by snapshot from index:%ld to index:%ld", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, snapshot.lastApplyIndex); } // update commit index diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ffa6dc1291..00786e5015 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -573,9 +573,9 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return -1; } assert(rid == pSyncNode->rid); - sDebug("vgId:%d sync event %s currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), - pMsg->msgType); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, + TMSG_INFO(pMsg->msgType), pMsg->msgType); ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -584,9 +584,9 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; - sDebug("vgId:%d sync event %s currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), - pMsg->msgType); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, + TMSG_INFO(pMsg->msgType), pMsg->msgType); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -832,8 +832,8 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // snapshot meta // pSyncNode->sMeta.lastConfigIndex = -1; - sDebug("vgId:%d sync event %s currentTerm:%lu sync open", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), - pSyncNode->pRaftStore->currentTerm); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu sync open", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm); return pSyncNode; } @@ -880,8 +880,8 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodeClose(SSyncNode* pSyncNode) { - sDebug("vgId:%d sync event %s currentTerm:%lu sync close", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), - pSyncNode->pRaftStore->currentTerm); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu sync close", pSyncNode->vgId, + syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm); int32_t ret; assert(pSyncNode != NULL); @@ -1320,9 +1320,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { oldSenders[i] = (pSyncNode->senders)[i]; - sDebug("vgId:%d sync event %s currentTerm:%lu save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, i, oldSenders[i], - oldSenders[i]->privateTerm); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu save senders %d, %p, privateTerm:%lu", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, i, oldSenders[i], oldSenders[i]->privateTerm); if (gRaftDetailLog) { ; } @@ -1374,9 +1374,12 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event %s currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, privateTerm:%lu", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j], oldSenders[j]->privateTerm); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, " + "privateTerm:%lu", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j], + oldSenders[j]->privateTerm); (pSyncNode->senders)[i] = oldSenders[j]; oldSenders[j] = NULL; reset = true; @@ -1384,9 +1387,11 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex // reset replicaIndex int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; (pSyncNode->senders)[i]->replicaIndex = i; - sDebug("vgId:%d sync event %s currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, " + "reset:%d", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); } } } @@ -1395,9 +1400,10 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sDebug("vgId:%d sync event %s currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - (pSyncNode->senders)[i], i, (pSyncNode->senders)[i]->privateTerm); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, (pSyncNode->senders)[i], i, (pSyncNode->senders)[i]->privateTerm); } } @@ -1405,8 +1411,9 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { snapshotSenderDestroy(oldSenders[i]); - sDebug("vgId:%d sync event %s currentTerm:%lu delete old sender %p replicaIndex:%d", pSyncNode->vgId, - syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, oldSenders[i], i); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu delete old sender %p replicaIndex:%d", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, oldSenders[i], i); oldSenders[i] = NULL; } } @@ -1477,10 +1484,11 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { sDebug( - "vgId:%d sync event %s currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " "restoreFinish:%d, %s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->restoreFinish, debugStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->restoreFinish, debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1517,9 +1525,12 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // reset restoreFinish pSyncNode->restoreFinish = false; - sDebug("vgId:%d sync event %s currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, restoreFinish:%d, %s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->restoreFinish, debugStr); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, " + "restoreFinish:%d, %s", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, + pSyncNode->restoreFinish, debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -2093,14 +2104,15 @@ const char* syncStr(ESyncState state) { static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - sDebug("vgId:%d sync event %s currentTerm:%lu begin leader transfer", ths->vgId, syncUtilState2String(ths->state), - ths->pRaftStore->currentTerm); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu begin leader transfer", ths->vgId, + syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm); if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { - sDebug("vgId:%d sync event %s currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, - syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, - pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, + syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, + pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, + pSyncLeaderTransfer->newLeaderId.addr); // reset elect timer now! int32_t electMS = 1; @@ -2238,9 +2250,10 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; - sDebug("vgId:%d sync event %s currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", - ths->vgId, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, beginIndex, endIndex, - syncUtilState2String(state)); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 + ", %s", + ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, beginIndex, + endIndex, syncUtilState2String(state)); // execute fsm if (ths->pFsm != NULL) { @@ -2288,9 +2301,9 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sDebug("vgId:%d sync event %s currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, - syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, syncUtilState2String(ths->state), - pEntry->index); + sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, + syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, + syncUtilState2String(ths->state), pEntry->index); } } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 308edc0b93..7c6d97a96e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -163,10 +163,12 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); - sDebug("vgId:%d sync event %s currentTerm:%lu write index:%ld, isStandBy:%d, msgType:%s,%d, originalRpcType:%s,%d", - pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), - pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu write index:%ld, isStandBy:%d, msgType:%s,%d, " + "originalRpcType:%s,%d", + pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->commitIndex, + pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); return code; } @@ -323,11 +325,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { walFsync(pWal, true); sDebug( - "vgId:%d sync event %s currentTerm:%lu old write index:%ld, isStandBy:%d, msgType:%s,%d, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu old write index:%ld, isStandBy:%d, msgType:%s,%d, " "originalRpcType:%s,%d", - pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->pRaftStore->currentTerm, - pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, TMSG_INFO(pEntry->msgType), pEntry->msgType, - TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->commitIndex, + pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); return code; } diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index cc4004a6bd..551c2611b0 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -46,10 +46,11 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event %s currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, + pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return keyCode; @@ -72,10 +73,12 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event %s currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p ahandle:%p", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p " + "ahandle:%p", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, + pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); taosThreadMutexUnlock(&(pObj->mutex)); return 1; // get one object @@ -92,10 +95,12 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu memcpy(pStub, pTmp, sizeof(SRespStub)); SSyncNode *pSyncNode = pObj->data; - sDebug("vgId:%d sync event %s currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p ahandle:%p", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, - pStub->rpcMsg.info.ahandle); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p " + "ahandle:%p", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, + pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); taosHashRemove(pObj->pRespHash, &index, sizeof(index)); taosThreadMutexUnlock(&(pObj->mutex)); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 68336499bf..ef2011cc26 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -141,21 +141,23 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d " + "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d " + "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); @@ -285,31 +287,34 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d " + "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d " + "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); } } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d lastApplyIndex:%ld " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d " + "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->privateTerm); @@ -344,16 +349,19 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu send " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d " + "privateTerm:%lu send " "msg:%s", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->privateTerm, msgStr); taosMemoryFree(msgStr); } else { - sDebug("vgId:%d sync event %s currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d privateTerm:%lu", - pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), - pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->privateTerm); + sDebug( + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d " + "privateTerm:%lu", + pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, + pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->privateTerm); } syncSnapshotSendDestroy(pMsg); @@ -586,19 +594,23 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d begin ack:%d, " + "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, - pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d begin ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d begin ack:%d, " + "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld privateTerm:%lu", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, - pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex, pReceiver->privateTerm); } } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { @@ -636,18 +648,20 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { bool isDrop; if (IamInNew) { sDebug( - "vgId:%d sync event %s currentTerm:%lu update config by snapshot, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu update config by snapshot, lastIndex:%ld, " + "lastTerm:%lu, " "lastConfigIndex:%ld ", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); syncNodeUpdateConfig(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex, &isDrop); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu do not update config by snapshot, I am not in newCfg, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu do not update config by snapshot, I am not in " + "newCfg, " "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld ", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, - pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex); } // change isStandBy to normal @@ -680,21 +694,23 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d finish, update log begin " + "index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu, raft log:%s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, - pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, - pReceiver->privateTerm, logSimpleStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, + snapshot.lastApplyTerm, snapshot.lastConfigIndex, pReceiver->privateTerm, logSimpleStr); taosMemoryFree(logSimpleStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d finish, update log begin index:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d finish, update log begin " + "index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, - pMsg->lastIndex + 1, snapshot.lastApplyIndex, snapshot.lastApplyTerm, snapshot.lastConfigIndex, - pReceiver->privateTerm); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, host, port, pMsg->lastIndex + 1, snapshot.lastApplyIndex, + snapshot.lastApplyTerm, snapshot.lastConfigIndex, pReceiver->privateTerm); } pReceiver->pWriter = NULL; @@ -705,17 +721,19 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d end ack:%d, " + "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d end ack:%d, lastIndex:%ld, lastTerm:%lu, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d end ack:%d, " + "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } @@ -732,20 +750,22 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d force close ack:%d, " + "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv " "msg:%s", - pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d force close ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d force close ack:%d, " + "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), + pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pReceiver->pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); } @@ -767,19 +787,23 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, " + "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, - pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex, pReceiver->privateTerm, msgStr); taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, lastIndex:%ld, " + "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, " + "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, host, port, - pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, pMsg->lastConfigIndex, pReceiver->privateTerm); + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, + pSyncNode->pRaftStore->currentTerm, host, port, pReceiver->ack, pMsg->lastIndex, pMsg->lastTerm, + pMsg->lastConfigIndex, pReceiver->privateTerm); } } else { From d3f6f07902792dc3d0d4cdd5a9ecaf73fcb7fb70 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 19:40:08 +0800 Subject: [PATCH 52/90] fix(sync): fake match when prevLogIndex less than commitIndex --- source/libs/sync/src/syncAppendEntries.c | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 1235e8eda6..4ebd279668 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -880,6 +880,40 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } } while (0); + // fake match + // + // condition1: + // preIndex <= my commit index + // + // operation: + // match my commit index + // no operation on log + do { + bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && + (pMsg->prevLogIndex <= ths->commitIndex); + if (condition) { + sTrace("recv SyncAppendEntries, fake match2, msg-prevLogIndex:%ld, my-commitIndex:%ld", pMsg->prevLogIndex, + ths->commitIndex); + + // prepare response msg + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; + pReply->success = true; + pReply->matchIndex = ths->commitIndex; + + // send response + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + } while (0); + // calculate logOK here, before will coredump, due to fake match bool logOK = syncNodeOnAppendEntriesLogOK(ths, pMsg); From f350ade453d35f2a037998d0015c83bcbe323c47 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 16 Jun 2022 19:45:09 +0800 Subject: [PATCH 53/90] feat: the unique function and tail function are rewritten as the corresponding clauses --- include/libs/nodes/querynodes.h | 3 +++ source/libs/function/src/builtins.c | 7 ++++++- source/libs/nodes/src/nodesUtilFuncs.c | 20 ++++++++++++++++++ source/libs/parser/src/parTranslater.c | 3 +++ tests/system-test/2-query/tail.py | 23 ++++++++++---------- tests/system-test/2-query/unique.py | 29 +++++++++++++------------- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 3adc619567..944b1f4b2e 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -376,6 +376,9 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp); bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsRegularOp(const SOperatorNode* pOp); +bool nodesExprHasColumn(SNode* pNode); +bool nodesExprsHasColumn(SNodeList* pList); + void* nodesGetValueFromNode(SValueNode* pNode); int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value); char* nodesGetStrValueFromNode(SValueNode* pNode); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1cfa4d201e..e1375e8a6f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1085,7 +1085,12 @@ static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (!nodesExprHasColumn(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE must contain columns"); + } + + pFunc->node.resType = ((SExprNode*)pPara)->resType; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index a45ba53ad1..e2f4dd26f6 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1463,6 +1463,26 @@ int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeT return TSDB_CODE_SUCCESS; } +static EDealRes hasColumn(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + *(bool*)pContext = true; + return DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +bool nodesExprHasColumn(SNode* pNode) { + bool hasCol = false; + nodesWalkExprPostOrder(pNode, hasColumn, &hasCol); + return hasCol; +} + +bool nodesExprsHasColumn(SNodeList* pList) { + bool hasCol = false; + nodesWalkExprsPostOrder(pList, hasColumn, &hasCol); + return hasCol; +} + char* nodesGetFillModeString(EFillMode mode) { switch (mode) { case FILL_MODE_NONE: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f17d91f042..814c247e09 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2170,6 +2170,7 @@ static EDealRes rewriteSeletcValueFunc(STranslateContext* pCxt, SNode** pNode) { } strcpy(pFirst->functionName, "first"); TSWAP(pFirst->pParameterList, ((SFunctionNode*)*pNode)->pParameterList); + strcpy(pFirst->node.aliasName, ((SExprNode*)*pNode)->aliasName); nodesDestroyNode(*pNode); *pNode = (SNode*)pFirst; pCxt->errCode = fmGetFuncInfo(pFirst, pCxt->msgBuf.buf, pCxt->msgBuf.len); @@ -2184,6 +2185,7 @@ static EDealRes rewriteUniqueFunc(SNode** pNode, void* pContext) { if (FUNCTION_TYPE_UNIQUE == pFunc->funcType) { SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); NODES_CLEAR_LIST(pFunc->pParameterList); + strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName); nodesDestroyNode(*pNode); *pNode = pExpr; pCxt->pExpr = pExpr; @@ -2238,6 +2240,7 @@ static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) { pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i; } SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0); + strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName); NODES_CLEAR_LIST(pFunc->pParameterList); nodesDestroyNode(*pNode); *pNode = pExpr; diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 6039f3effa..bb11fbbef5 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -188,7 +188,8 @@ class TDTestCase: def check_tail_table(self , tbname , col_name , tail_rows , offset): tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}" - equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" tdSql.query(tail_sql) tail_result = tdSql.queryResult @@ -294,21 +295,21 @@ class TDTestCase: tdSql.checkData(1, 0, None) tdSql.query("select tail(c1,3,2) from ct4 where c1 >2 ") - tdSql.checkData(0, 0, 7) + tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 6) - tdSql.checkData(2, 0, 5) + tdSql.checkData(2, 0, 7) tdSql.query("select tail(c1,2,1) from ct4 where c2 between 0 and 99999") - tdSql.checkData(0, 0, 2) - tdSql.checkData(1, 0, 1) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 2) # tail with union all tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct1") tdSql.checkRows(15) tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct2") tdSql.checkRows(2) - tdSql.checkData(0, 0, 1) - tdSql.checkData(1, 0, 0) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 1) tdSql.query("select tail(c2,2,1) from ct4 union all select abs(c2)/2 from ct4") tdSql.checkRows(14) @@ -336,16 +337,16 @@ class TDTestCase: tdSql.query("select tail(tb2.num,3,2) from tb1, tb2 where tb1.ts=tb2.ts ") tdSql.checkRows(3) - tdSql.checkData(0,0,5) + tdSql.checkData(0,0,7) tdSql.checkData(1,0,6) - tdSql.checkData(2,0,7) + tdSql.checkData(2,0,5) # nest query # tdSql.query("select tail(c1,2) from (select c1 from ct1)") tdSql.query("select c1 from (select tail(c1,2) c1 from ct4)") tdSql.checkRows(2) - tdSql.checkData(0, 0, 0) - tdSql.checkData(1, 0, None) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 0) tdSql.query("select sum(c1) from (select tail(c1,2) c1 from ct1)") tdSql.checkRows(1) diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 227efa6f9c..f48202010a 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -93,8 +93,8 @@ class TDTestCase: "select unique(c1) , min(c1) from t1", "select unique(c1) , spread(c1) from t1", "select unique(c1) , diff(c1) from t1", - "select unique(c1) , abs(c1) from t1", - "select unique(c1) , c1 from t1", + #"select unique(c1) , abs(c1) from t1", # support + #"select unique(c1) , c1 from t1", "select unique from stb1 partition by tbname", "select unique(123--123)==1 from stb1 partition by tbname", "select unique(123) from stb1 partition by tbname", @@ -104,21 +104,21 @@ class TDTestCase: "select unique(c1 ,c2 ) from stb1 partition by tbname", "select unique(c1 ,NULL) from stb1 partition by tbname", "select unique(,) from stb1 partition by tbname;", - "select unique(floor(c1) ab from stb1 partition by tbname)", - "select unique(c1) as int from stb1 partition by tbname", + #"select unique(floor(c1) ab from stb1 partition by tbname)", # support + #"select unique(c1) as int from stb1 partition by tbname", "select unique('c1') from stb1 partition by tbname", "select unique(NULL) from stb1 partition by tbname", "select unique('') from stb1 partition by tbname", "select unique(c%) from stb1 partition by tbname", - #"select unique(t1) from stb1 partition by tbname", + #"select unique(t1) from stb1 partition by tbname", # support "select unique(True) from stb1 partition by tbname", "select unique(c1) , count(c1) from stb1 partition by tbname", "select unique(c1) , avg(c1) from stb1 partition by tbname", "select unique(c1) , min(c1) from stb1 partition by tbname", "select unique(c1) , spread(c1) from stb1 partition by tbname", "select unique(c1) , diff(c1) from stb1 partition by tbname", - "select unique(c1) , abs(c1) from stb1 partition by tbname", - "select unique(c1) , c1 from stb1 partition by tbname" + #"select unique(c1) , abs(c1) from stb1 partition by tbname", # support + #"select unique(c1) , c1 from stb1 partition by tbname" # support ] for error_sql in error_sql_lists: @@ -198,7 +198,7 @@ class TDTestCase: unique_datas = [] for elem in unique_result: unique_datas.append(elem[0]) - + unique_datas.sort(key=lambda x: (x is None, x)) tdSql.query(origin_sql) origin_result = tdSql.queryResult @@ -212,6 +212,7 @@ class TDTestCase: continue else: pre_unique.append(elem) + pre_unique.sort(key=lambda x: (x is None, x)) if pre_unique == unique_datas: tdLog.info(" unique query check pass , unique sql is: %s" %unique_sql) @@ -266,16 +267,16 @@ class TDTestCase: tdSql.checkRows(10) tdSql.error("select unique(c1),tbname from ct1") - tdSql.error("select unique(c1),t1 from ct1") + #tdSql.error("select unique(c1),t1 from ct1") #support # unique with common col - tdSql.error("select unique(c1) ,ts from ct1") - tdSql.error("select unique(c1) ,c1 from ct1") + #tdSql.error("select unique(c1) ,ts from ct1") + #tdSql.error("select unique(c1) ,c1 from ct1") # unique with scalar function - tdSql.error("select unique(c1) ,abs(c1) from ct1") + #tdSql.error("select unique(c1) ,abs(c1) from ct1") tdSql.error("select unique(c1) , unique(c2) from ct1") - tdSql.error("select unique(c1) , abs(c2)+2 from ct1") + #tdSql.error("select unique(c1) , abs(c2)+2 from ct1") # unique with aggregate function @@ -288,7 +289,7 @@ class TDTestCase: tdSql.query("select unique(c1) from ct4 where c1 is null") tdSql.checkData(0, 0, None) - tdSql.query("select unique(c1) from ct4 where c1 >2 ") + tdSql.query("select unique(c1) from ct4 where c1 >2") tdSql.checkData(0, 0, 8) tdSql.checkData(1, 0, 7) tdSql.checkData(2, 0, 6) From 67540f2504464927ff1f684f97481138bfc19baf Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 16 Jun 2022 19:49:42 +0800 Subject: [PATCH 54/90] fix: table merge scan sort buf size --- source/libs/executor/src/scanoperator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ddebba1c9e..a8aad8fe68 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2310,14 +2310,14 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pSortInfo = generateSortByTsInfo(pInfo->cond.order); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); int32_t rowSize = pInfo->pResBlock->info.rowSize; - pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; + int32_t blockMetaSize = (int32_t)blockDataGetSerialMetaSize(pInfo->pResBlock->info.numOfCols); + pInfo->bufPageSize = (rowSize * 2 + blockMetaSize) < 1024 ? 1024 : (rowSize * 2 + blockMetaSize); pInfo->sortBufSize = pInfo->bufPageSize * 16; pInfo->hasGroupId = false; pInfo->prefetchedTuple = NULL; pOperator->name = "TableMergeScanOperator"; - // TODO : change it - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; From 98cbbc01975f78132c0ac90c0d0c1be6c73ef507 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 16 Jun 2022 19:53:39 +0800 Subject: [PATCH 55/90] test: add auto exit case --- tests/system-test/test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 5a68f548da..f3e39fc9ce 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -13,6 +13,7 @@ # pip install src/connector/python/ # -*- coding: utf-8 -*- +from ast import If import sys import getopt import subprocess @@ -33,8 +34,15 @@ import taos def checkRunTimeError(): import win32gui + timeCount = 0 while 1: time.sleep(1) + timeCount = timeCount + 1 + if (timeCount>900): + os.system("TASKKILL /F /IM taosd.exe") + os.system("TASKKILL /F /IM taos.exe") + os.system("TASKKILL /F /IM tmq_sim.exe") + quit(0) hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") if hwnd: os.system("TASKKILL /F /IM taosd.exe") From c858b6952853677f3e2a25d5532b63a5114f7df6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 16 Jun 2022 19:56:51 +0800 Subject: [PATCH 56/90] test: add auto exit case --- tests/system-test/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index f3e39fc9ce..85f63a2868 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -13,7 +13,6 @@ # pip install src/connector/python/ # -*- coding: utf-8 -*- -from ast import If import sys import getopt import subprocess From 49d92079ca1005546eca0b87daba8fbdaee3e509 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Thu, 16 Jun 2022 19:58:24 +0800 Subject: [PATCH 57/90] test: add auto exit case --- tests/system-test/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 85f63a2868..55bfd8c945 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -41,6 +41,7 @@ def checkRunTimeError(): os.system("TASKKILL /F /IM taosd.exe") os.system("TASKKILL /F /IM taos.exe") os.system("TASKKILL /F /IM tmq_sim.exe") + os.system("TASKKILL /F /IM mintty.exe") quit(0) hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library") if hwnd: From b42c963a5f7085b7dac21076de7996a33d187ce2 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 20:08:27 +0800 Subject: [PATCH 58/90] fix(sync): fake match when prevLogIndex less than commitIndex --- source/libs/sync/src/syncAppendEntries.c | 26 +++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4ebd279668..7a4edb668a 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -880,13 +880,14 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } } while (0); - // fake match + // fake match2 // // condition1: // preIndex <= my commit index // // operation: - // match my commit index + // if hasAppendEntries && pMsg->prevLogIndex == ths->commitIndex, append entry + // match my-commit-index or my-commit-index + 1 // no operation on log do { bool condition = (pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && @@ -895,6 +896,25 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs sTrace("recv SyncAppendEntries, fake match2, msg-prevLogIndex:%ld, my-commitIndex:%ld", pMsg->prevLogIndex, ths->commitIndex); + SyncIndex matchIndex = ths->commitIndex; + bool hasAppendEntries = pMsg->dataLen > 0; + if (hasAppendEntries && pMsg->prevLogIndex == ths->commitIndex) { + // append entry + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + ASSERT(pAppendEntry != NULL); + + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); + ASSERT(code == 0); + + // pre commit + code = syncNodePreCommit(ths, pAppendEntry); + ASSERT(code == 0); + + matchIndex = pMsg->prevLogIndex + 1; + + syncEntryDestory(pAppendEntry); + } + // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; @@ -902,7 +922,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->term = ths->pRaftStore->currentTerm; pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = true; - pReply->matchIndex = ths->commitIndex; + pReply->matchIndex = matchIndex; // send response SRpcMsg rpcMsg; From bdf24b6ab06c991db5198835d18fff3720f793f1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 16 Jun 2022 20:14:48 +0800 Subject: [PATCH 59/90] fix: uncomment the codes of dropping stable --- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index d170931e7e..da1a126a76 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -466,10 +466,10 @@ static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pRe } // process request - // if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { - // rcode = terrno; - // goto _exit; - // } + if (metaDropSTable(pVnode->pMeta, version, &req) < 0) { + rcode = terrno; + goto _exit; + } // return rsp _exit: From 8353da6694408895df7f277aa69920968dc560ce Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 16 Jun 2022 20:17:36 +0800 Subject: [PATCH 60/90] other: revert the code --- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index e4c5ef8fc3..1c2514d46f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -233,17 +233,6 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { tsdbError("vgId:%d, SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len); return -1; - } else { - SBlockInfo *pBlkInfo = pReadh->pBlkInfo; - SBlock *pBlk = NULL; - int nBlks = (pBlkIdx->len - sizeof(SBlockInfo)) / sizeof(SBlock); - for (int n = 0; n < nBlks; ++n) { - pBlk = &pBlkInfo->blocks[n]; - if (pBlk->numOfSubBlocks >= 1) { - tsdbDebug("prop:vgId:%d, file %s , offset:%u len :%u has %d rows", TSDB_READ_REPO_ID(pReadh), - TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, pBlk->numOfRows); - } - } } // ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid); From d71027ac359a555f9f8d035ffa80ec80aa6566d9 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 16 Jun 2022 20:43:49 +0800 Subject: [PATCH 61/90] fix(sync): fake match2, need truncate log --- source/libs/sync/src/syncAppendEntries.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7a4edb668a..7b2f79e24c 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -903,6 +903,18 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); ASSERT(pAppendEntry != NULL); + { + // has extra entries (> preIndex) in local log + SyncIndex logLastIndex = ths->pLogStore->syncLogLastIndex(ths->pLogStore); + bool hasExtraEntries = logLastIndex > pMsg->prevLogIndex; + + if (hasExtraEntries) { + // make log same, rollback deleted entries + code = syncNodeMakeLogSame(ths, pMsg); + ASSERT(code == 0); + } + } + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); ASSERT(code == 0); From b3f9f81b6101277aa1cd2962ff18e740ca160884 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 16 Jun 2022 21:06:04 +0800 Subject: [PATCH 62/90] support show apps --- include/common/tmsg.h | 2 - include/libs/qcom/query.h | 2 - include/libs/scheduler/scheduler.h | 2 +- include/util/taoserror.h | 2 + source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 2 +- source/client/src/clientHb.c | 3 +- source/client/src/clientImpl.c | 8 +- source/client/src/clientMain.c | 19 +- source/common/src/systable.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 8 +- source/dnode/mnode/impl/src/mndProfile.c | 21 +- .../dnode/mnode/impl/test/profile/profile.cpp | 3 +- source/libs/executor/src/scanoperator.c | 2 + source/libs/parser/src/parAstCreater.c | 2 +- source/libs/qcom/src/queryUtil.c | 4 - source/libs/qworker/src/qwDbg.c | 18 +- source/libs/scheduler/inc/schedulerInt.h | 24 +- source/libs/scheduler/src/schJob.c | 310 +++++++++--------- source/libs/scheduler/src/schRemote.c | 2 - source/libs/scheduler/src/schUtil.c | 12 + source/libs/scheduler/src/scheduler.c | 41 ++- source/libs/scheduler/test/schedulerTests.cpp | 10 +- source/util/src/terror.c | 2 + tests/script/tsim/show/basic.sim | 6 +- 25 files changed, 277 insertions(+), 233 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c2a274045b..44316fcb69 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1315,8 +1315,6 @@ int32_t tSerializeSSetStandbyReq(void* buf, int32_t bufLen, SSetStandbyReq* pReq int32_t tDeserializeSSetStandbyReq(void* buf, int32_t bufLen, SSetStandbyReq* pReq); typedef struct { - int32_t connId; // todo remove - int32_t queryId; // todo remove char queryStrId[TSDB_QUERY_ID_LEN]; } SKillQueryReq; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 89eb21556a..b6a3a08f7e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -33,8 +33,6 @@ typedef enum { JOB_TASK_STATUS_PARTIAL_SUCCEED, JOB_TASK_STATUS_SUCCEED, JOB_TASK_STATUS_FAILED, - JOB_TASK_STATUS_CANCELLING, - JOB_TASK_STATUS_CANCELLED, JOB_TASK_STATUS_DROPPING, } EJobTaskType; diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 8087a6d33c..dfff17221d 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -127,7 +127,7 @@ void schedulerStopQueryHb(void *pTrans); * Free the query job * @param pJob */ -void schedulerFreeJob(int64_t job); +void schedulerFreeJob(int64_t job, int32_t errCode); void schedulerDestroy(void); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4f93ce9e4a..16f608ffe6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -127,6 +127,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_STMT_API_ERROR TAOS_DEF_ERROR_CODE(0, 0X0225) #define TSDB_CODE_TSC_STMT_TBNAME_ERROR TAOS_DEF_ERROR_CODE(0, 0X0226) #define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0227) +#define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X0228) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -570,6 +571,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) #define TSDB_CODE_SCH_IGNORE_ERROR TAOS_DEF_ERROR_CODE(0, 0x2503) #define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504) +#define TSDB_CODE_SCH_JOB_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x2505) #define TSDB_CODE_QW_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x2550) //parser diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 98326ae830..d61262c9dc 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -204,6 +204,7 @@ typedef struct SRequestObj { SRequestSendRecvBody body; bool stableQuery; + bool killed; uint32_t prevCode; //previous error code: todo refactor, add update flag for catalog uint32_t retry; } SRequestObj; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 3b74554e76..24246e5c45 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -229,7 +229,7 @@ static void doDestroyRequest(void *p) { taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); + schedulerFreeJob(pRequest->body.queryJob, 0); } taosMemoryFreeClear(pRequest->msgBuf); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index ac9bca7295..4816b85587 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -164,6 +164,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->connId = pRsp->query->connId; if (pRsp->query->killRid) { + tscDebug("request rid %" PRIx64 " need to be killed now", pRsp->query->killRid); SRequestObj *pRequest = acquireRequest(pRsp->query->killRid); if (NULL == pRequest) { tscDebug("request 0x%" PRIx64 " not exist to kill", pRsp->query->killRid); @@ -304,7 +305,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { while (pIter != NULL) { int64_t *rid = pIter; SRequestObj *pRequest = acquireRequest(*rid); - if (NULL == pRequest) { + if (NULL == pRequest || pRequest->killed) { pIter = taosHashIterate(pObj->pRequests, pIter); continue; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 10c2161478..a9dd1bb50b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -418,7 +418,7 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod while (true) { if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); + schedulerFreeJob(pRequest->body.queryJob, 0); } pRequest->code = code; @@ -439,7 +439,7 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod pRequest->body.resInfo.numOfRows = res.numOfRows; if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); + schedulerFreeJob(pRequest->body.queryJob, 0); } } @@ -468,7 +468,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); + schedulerFreeJob(pRequest->body.queryJob, 0); } pRequest->code = code; @@ -481,7 +481,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList pRequest->body.resInfo.numOfRows = res.numOfRows; if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); + schedulerFreeJob(pRequest->body.queryJob, 0); } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 13a5acb472..2111ff4508 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -246,13 +246,14 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { if (TD_RES_QUERY(res)) { SRequestObj *pRequest = (SRequestObj *)res; + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || + pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0 || pRequest->killed) { + return NULL; + } + #if SYNC_ON_TOP_OF_ASYNC return doAsyncFetchRows(pRequest, true, true); #else - if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT || - pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) { - return NULL; - } return doFetchRows(pRequest, true, true); #endif @@ -482,14 +483,20 @@ void taos_stop_query(TAOS_RES *res) { } SRequestObj *pRequest = (SRequestObj *)res; + pRequest->killed = true; + int32_t numOfFields = taos_num_fields(pRequest); - // It is not a query, no need to stop. if (numOfFields == 0) { + tscDebug("request %" PRIx64 " no need to be killed since not query", pRequest->requestId); return; } - schedulerFreeJob(pRequest->body.queryJob); + if (pRequest->body.queryJob) { + schedulerFreeJob(pRequest->body.queryJob, TSDB_CODE_TSC_QUERY_KILLED); + } + + tscDebug("request %" PRIx64 " killed", pRequest->requestId); } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 38b223b2ac..192d1d9922 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -77,7 +77,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "duration", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "duration", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "keep", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -302,7 +302,7 @@ static const SSysDbTableSchema offsetSchema[] = { }; static const SSysDbTableSchema querySchema[] = { - {.name = "query_id", .bytes = 26 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c12c6d5b4c..88afd57e04 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1391,11 +1391,13 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)strict, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.daysPerFile, false); - char tmp[128] = {0}; int32_t len = 0; + len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.daysPerFile); + varDataSetLen(tmp, len); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)tmp, false); + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { len = sprintf(&tmp[VARSTR_HEADER_SIZE], "%dm,%dm,%dm", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index b0c6e57687..acc5270ce9 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -634,16 +634,27 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { return -1; } - mInfo("kill query msg is received, queryId:%d", killReq.queryId); + mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); + int32_t connId = 0; + uint64_t queryId = 0; + char* p = strchr(killReq.queryStrId, ':'); + if (NULL == p) { + mError("invalid query id %s", killReq.queryStrId); + terrno = TSDB_CODE_MND_INVALID_QUERY_ID; + return -1; + } + *p = 0; + connId = taosStr2Int32(killReq.queryStrId, NULL, 16); + queryId = taosStr2UInt64(p + 1, NULL, 16); - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(int32_t)); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%d, failed to kill queryId:%d, conn not exist", killReq.connId, killReq.queryId); + mError("connId:%x, failed to kill queryId:%" PRIx64 ", conn not exist", connId, queryId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, queryId:%d is killed by user:%s", killReq.connId, killReq.queryId, pReq->conn.user); - pConn->killId = killReq.queryId; + mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->conn.user); + pConn->killId = queryId; taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); return 0; } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 039700ae9e..e784a41d6f 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -295,8 +295,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) { SKillQueryReq killReq = {0}; - killReq.connId = 2345; - killReq.queryId = 2345; + strcpy(killReq.queryStrId, "2345:2345"); int32_t contLen = tSerializeSKillQueryReq(NULL, 0, &killReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0823594858..3e304ef14e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -537,6 +537,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } + //taosSsleep(20); + SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; int32_t numOfCols = 0; diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 8a7987e4ec..2bf345e7f7 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1449,7 +1449,7 @@ SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId) { CHECK_PARSER_STATUS(pCxt); SKillQueryStmt* pStmt = (SKillQueryStmt*)nodesMakeNode(QUERY_NODE_KILL_QUERY_STMT); CHECK_OUT_OF_MEM(pStmt); - strncpy(pStmt->queryId, pQueryId->z, TMIN(pQueryId->n, sizeof(pStmt->queryId) - 1)); + trimString(pQueryId->z, pQueryId->n, pStmt->queryId, sizeof(pStmt->queryId) - 1); return (SNode*)pStmt; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index a5a499aaf5..9322d7e594 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -177,10 +177,6 @@ char* jobTaskStatusStr(int32_t status) { return "SUCCEED"; case JOB_TASK_STATUS_FAILED: return "FAILED"; - case JOB_TASK_STATUS_CANCELLING: - return "CANCELLING"; - case JOB_TASK_STATUS_CANCELLED: - return "CANCELLED"; case JOB_TASK_STATUS_DROPPING: return "DROPPING"; default: diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 368c3bb517..add9700a3a 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -44,40 +44,30 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, break; case JOB_TASK_STATUS_EXECUTING: if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_SUCCEED && - newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_CANCELLING && - newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING) { + newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_PARTIAL_SUCCEED: if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_SUCCEED && - newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_FAILED && - newStatus != JOB_TASK_STATUS_DROPPING) { + newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_SUCCEED: - if (newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING && - newStatus != JOB_TASK_STATUS_FAILED) { + if (newStatus != JOB_TASK_STATUS_DROPPING && newStatus != JOB_TASK_STATUS_FAILED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; case JOB_TASK_STATUS_FAILED: - if (newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; - case JOB_TASK_STATUS_CANCELLING: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: if (newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 8875ebdf73..2a3a069d02 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -48,6 +48,12 @@ enum { SCH_FETCH_CB, }; +typedef enum { + SCH_OP_NULL = 0, + SCH_OP_EXEC, + SCH_OP_FETCH, +} SCH_OP_TYPE; + typedef struct SSchTrans { void *pTrans; void *pHandle; @@ -188,11 +194,15 @@ typedef struct SSchTask { typedef struct SSchJobAttr { EExplainMode explainMode; - bool syncSchedule; bool queryJob; bool needFlowCtrl; } SSchJobAttr; +typedef struct { + int32_t op; + bool sync; +} SSchOpStatus; + typedef struct SSchJob { int64_t refId; uint64_t queryId; @@ -217,8 +227,7 @@ typedef struct SSchJob { int8_t status; SQueryNodeAddr resNode; tsem_t rspSem; - int8_t userFetch; - int32_t remoteFetch; + SSchOpStatus opStatus; SSchTask *fetchTask; int32_t errCode; SRWLatch resLock; @@ -227,7 +236,6 @@ typedef struct SSchJob { int32_t resNumOfRows; SSchResInfo userRes; const char *sql; - int32_t userCb; SQueryProfileSummary summary; } SSchJob; @@ -285,6 +293,10 @@ extern SSchedulerMgmt schMgmt; #define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status) #define SCH_GET_JOB_STATUS_STR(job) jobTaskStatusStr(SCH_GET_JOB_STATUS(job)) +#define SCH_JOB_IN_SYNC_OP(job) ((job)->opStatus.op && (job)->opStatus.sync) +#define SCH_JOB_IN_ASYNC_EXEC_OP(job) (((job)->opStatus.op == SCH_OP_EXEC) && (!(job)->opStatus.sync)) +#define SCH_JOB_IN_ASYNC_FETCH_OP(job) (((job)->opStatus.op == SCH_OP_FETCH) && (!(job)->opStatus.sync)) + #define SCH_SET_JOB_NEED_FLOW_CTRL(_job) (_job)->attr.needFlowCtrl = true #define SCH_JOB_NEED_FLOW_CTRL(_job) ((_job)->attr.needFlowCtrl) #define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_SRC_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level)) @@ -356,7 +368,7 @@ int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal * int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execIdx); int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bool sync); -int32_t schChkUpdateJobStatus(SSchJob *pJob, int8_t newStatus); +int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); int32_t schCancelJob(SSchJob *pJob); int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); uint64_t schGenTaskId(void); @@ -368,6 +380,8 @@ int32_t schAsyncFetchRows(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execIdx); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); +char* schGetOpStr(SCH_OP_TYPE type); +int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 3e3f3639d7..d723a431c2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -52,7 +52,6 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b } pJob->attr.explainMode = pReq->pDag->explainInfo.mode; - pJob->attr.syncSchedule = syncSchedule; pJob->conn = *pReq->pConn; pJob->sql = pReq->sql; pJob->userRes.queryRes = pRes; @@ -161,12 +160,11 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { *pStatus = status; } - return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || - status == JOB_TASK_STATUS_CANCELLING || status == JOB_TASK_STATUS_DROPPING || + return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_DROPPING || status == JOB_TASK_STATUS_SUCCEED); } -int32_t schChkUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { +int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; int8_t oriStatus = 0; @@ -175,7 +173,11 @@ int32_t schChkUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { oriStatus = SCH_GET_JOB_STATUS(pJob); if (oriStatus == newStatus) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + if (newStatus == JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_SCH_JOB_IS_DROPPING); + } + + SCH_ERR_JRET(TSDB_CODE_SCH_IGNORE_ERROR); } switch (oriStatus) { @@ -193,7 +195,6 @@ int32_t schChkUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_EXECUTING: if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_FAILED && - newStatus != JOB_TASK_STATUS_CANCELLING && newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -208,13 +209,11 @@ int32_t schChkUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_SUCCEED: case JOB_TASK_STATUS_FAILED: - case JOB_TASK_STATUS_CANCELLING: if (newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; - case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); break; @@ -238,8 +237,65 @@ int32_t schChkUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { _return: SCH_JOB_ELOG("invalid job status update, from %s to %s", jobTaskStatusStr(oriStatus), jobTaskStatusStr(newStatus)); - SCH_ERR_RET(code); + SCH_RET(code); +} + + +void schEndOperation(SSchJob *pJob) { + int32_t op = atomic_load_32(&pJob->opStatus.op); + if (SCH_OP_NULL == op) { + SCH_JOB_DLOG("job already not in any operation, status:%s", jobTaskStatusStr(pJob->status)); + return; + } + + atomic_store_32(&pJob->opStatus.op, SCH_OP_NULL); + + SCH_JOB_DLOG("job end %s operation", schGetOpStr(op)); +} + +int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync) { + int32_t code = 0; + + if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { + SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); + SCH_ERR_JRET(TSDB_CODE_TSC_APP_ERROR); + } + + SCH_JOB_ELOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); + + pJob->opStatus.sync = sync; + + switch (type) { + case SCH_OP_EXEC: + SCH_ERR_JRET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); + break; + case SCH_OP_FETCH: + if (!SCH_JOB_NEED_FETCH(pJob)) { + SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + int8_t status = 0; + if (schJobNeedToStop(pJob, &status)) { + SCH_JOB_ELOG("job need to stop cause of status %s", jobTaskStatusStr(status)); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + } else if (status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); + } + break; + default: + SCH_JOB_ELOG("unknown operation type %d", type); + SCH_ERR_JRET(TSDB_CODE_TSC_APP_ERROR); + } + return TSDB_CODE_SUCCESS; + +_return: + + schEndOperation(pJob); + + SCH_RET(code); } int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { @@ -828,7 +884,7 @@ int32_t schSetJobQueryRes(SSchJob* pJob, SQueryResult* pRes) { int32_t schSetJobFetchRes(SSchJob* pJob, void** pData) { int32_t code = 0; if (pJob->resData && ((SRetrieveTableRsp *)pJob->resData)->completed) { - SCH_ERR_RET(schChkUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); + SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_SUCCEED)); } while (true) { @@ -855,15 +911,17 @@ int32_t schSetJobFetchRes(SSchJob* pJob, void** pData) { return TSDB_CODE_SUCCESS; } -int32_t schNotifyUserQueryRes(SSchJob* pJob) { - pJob->userRes.queryRes = taosMemoryCalloc(1, sizeof(*pJob->userRes.queryRes)); - if (pJob->userRes.queryRes) { - schSetJobQueryRes(pJob, pJob->userRes.queryRes); +int32_t schNotifyUserExecRes(SSchJob* pJob) { + SQueryResult* pRes = taosMemoryCalloc(1, sizeof(SQueryResult)); + if (pRes) { + schSetJobQueryRes(pJob, pRes); } - (*pJob->userRes.execFp)(pJob->userRes.queryRes, pJob->userRes.userParam, atomic_load_32(&pJob->errCode)); + schEndOperation(pJob); - pJob->userRes.queryRes = NULL; + SCH_JOB_DLOG("sch start to invoke exec cb, code: %s", tstrerror(pJob->errCode)); + (*pJob->userRes.execFp)(pRes, pJob->userRes.userParam, atomic_load_32(&pJob->errCode)); + SCH_JOB_DLOG("sch end from query cb, code: %s", tstrerror(pJob->errCode)); return TSDB_CODE_SUCCESS; } @@ -871,35 +929,49 @@ int32_t schNotifyUserQueryRes(SSchJob* pJob) { int32_t schNotifyUserFetchRes(SSchJob* pJob) { void* pRes = NULL; - SCH_ERR_RET(schSetJobFetchRes(pJob, &pRes)); + schSetJobFetchRes(pJob, &pRes); + schEndOperation(pJob); + + SCH_JOB_DLOG("sch start to invoke fetch cb, code: %s", tstrerror(pJob->errCode)); (*pJob->userRes.fetchFp)(pRes, pJob->userRes.userParam, atomic_load_32(&pJob->errCode)); + SCH_JOB_DLOG("sch end from fetch cb, code: %s", tstrerror(pJob->errCode)); return TSDB_CODE_SUCCESS; } +void schPostJobRes(SSchJob *pJob, SCH_OP_TYPE op) { + if (SCH_OP_NULL == pJob->opStatus.op) { + SCH_JOB_DLOG("job not in any op, no need to post job res, status:%s", jobTaskStatusStr(pJob->status)); + return; + } + + if (op && pJob->opStatus.op != op) { + SCH_JOB_ELOG("job in op %s mis-match with expected %s", schGetOpStr(pJob->opStatus.op), schGetOpStr(op)); + return; + } + + if (SCH_JOB_IN_SYNC_OP(pJob)) { + tsem_post(&pJob->rspSem); + } else if (SCH_JOB_IN_ASYNC_EXEC_OP(pJob)) { + schNotifyUserExecRes(pJob); + } else if (SCH_JOB_IN_ASYNC_FETCH_OP(pJob)) { + schNotifyUserFetchRes(pJob); + } else { + SCH_JOB_ELOG("job not in any operation, status:%s", jobTaskStatusStr(pJob->status)); + } +} + int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCode) { // if already FAILED, no more processing - SCH_ERR_RET(schChkUpdateJobStatus(pJob, status)); + SCH_ERR_RET(schUpdateJobStatus(pJob, status)); schUpdateJobErrCode(pJob, errCode); - - if (atomic_load_8(&pJob->userFetch) || pJob->attr.syncSchedule) { - tsem_post(&pJob->rspSem); - } - + int32_t code = atomic_load_32(&pJob->errCode); - SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); - if (!pJob->attr.syncSchedule) { - if (SCH_EXEC_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_EXEC_CB, 0)) { - schNotifyUserQueryRes(pJob); - } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_FETCH_CB, 0)) { - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - schNotifyUserFetchRes(pJob); - } - } + schPostJobRes(pJob, 0); SCH_RET(code); } @@ -918,20 +990,9 @@ int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode) { int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t code = 0; - SCH_ERR_RET(schChkUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); + SCH_ERR_RET(schUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); - if (pJob->attr.syncSchedule) { - tsem_post(&pJob->rspSem); - } else if (SCH_EXEC_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_EXEC_CB, 0)) { - schNotifyUserQueryRes(pJob); - } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&pJob->userCb, SCH_FETCH_CB, 0)) { - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - schNotifyUserFetchRes(pJob); - } - - if (atomic_load_8(&pJob->userFetch)) { - SCH_ERR_JRET(schFetchFromRemote(pJob)); - } + schPostJobRes(pJob, SCH_OP_EXEC); return TSDB_CODE_SUCCESS; @@ -940,16 +1001,8 @@ _return: SCH_RET(schProcessOnJobFailure(pJob, code)); } -void schProcessOnDataFetched(SSchJob *job) { - atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); - - if (job->attr.syncSchedule) { - tsem_post(&job->rspSem); - } else if (SCH_FETCH_CB == atomic_val_compare_exchange_32(&job->userCb, SCH_FETCH_CB, 0)) { - atomic_val_compare_exchange_8(&job->userFetch, 1, 0); - - schNotifyUserFetchRes(job); - } +void schProcessOnDataFetched(SSchJob *pJob) { + schPostJobRes(pJob, SCH_OP_FETCH); } // Note: no more task error processing, handled in function internal @@ -1127,15 +1180,8 @@ _return: int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; - if (atomic_val_compare_exchange_32(&pJob->remoteFetch, 0, 1) != 0) { - SCH_JOB_ELOG("prior fetching not finished, remoteFetch:%d", atomic_load_32(&pJob->remoteFetch)); - return TSDB_CODE_SUCCESS; - } - void *resData = atomic_load_ptr(&pJob->resData); if (resData) { - atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); - SCH_JOB_DLOG("res already fetched, res:%p", resData); return TSDB_CODE_SUCCESS; } @@ -1146,8 +1192,6 @@ int32_t schFetchFromRemote(SSchJob *pJob) { _return: - atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); - SCH_RET(schProcessOnTaskFailure(pJob, pJob->fetchTask, code)); } @@ -1382,8 +1426,6 @@ int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level) { int32_t schLaunchJob(SSchJob *pJob) { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); - SCH_ERR_RET(schChkUpdateJobStatus(pJob, JOB_TASK_STATUS_EXECUTING)); - SCH_ERR_RET(schChkJobNeedFlowCtrl(pJob, level)); SCH_ERR_RET(schLaunchLevelTasks(pJob, level)); @@ -1483,24 +1525,36 @@ int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bo int32_t code = 0; SSchJob *pJob = NULL; - SCH_ERR_RET(schInitJob(pReq, &pJob, pRes, sync)); + SCH_ERR_JRET(schInitJob(pReq, &pJob, pRes, sync)); - qDebug("QID:0x%" PRIx64 " job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); + qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); *job = pJob->refId; - SCH_ERR_JRET(schLaunchJob(pJob)); + SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync)); + + code = schLaunchJob(pJob); if (sync) { SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); tsem_wait(&pJob->rspSem); - } else { - pJob->userCb = SCH_EXEC_CB; + + schEndOperation(pJob); + } else if (code) { + schPostJobRes(pJob, SCH_OP_EXEC); } - SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%"PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); + SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); + + schReleaseJob(pJob->refId); + + SCH_RET(code); _return: + if (!sync) { + pReq->fp(NULL, pReq->cbParam, code); + } + schReleaseJob(pJob->refId); SCH_RET(code); @@ -1534,10 +1588,10 @@ int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) { *pJob = 0; if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) { - SCH_ERR_RET(schExecStaticExplainJob(pReq, pJob, false)); - } else { - SCH_ERR_RET(schExecJobImpl(pReq, pJob, NULL, false)); + SCH_RET(schExecStaticExplainJob(pReq, pJob, false)); } + + SCH_ERR_RET(schExecJobImpl(pReq, pJob, NULL, false)); return code; } @@ -1549,17 +1603,24 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pReq->fp(NULL, pReq->cbParam, code); + SCH_ERR_RET(code); } pJob->sql = pReq->sql; pJob->attr.queryJob = true; - pJob->attr.syncSchedule = sync; pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->queryId = pReq->pDag->queryId; pJob->subPlans = pReq->pDag->pSubplans; pJob->userRes.execFp = pReq->fp; pJob->userRes.userParam = pReq->cbParam; + + code = schBeginOperation(pJob, SCH_OP_EXEC, sync); + if (code) { + pReq->fp(NULL, pReq->cbParam, code); + SCH_ERR_RET(code); + } SCH_ERR_JRET(qExecStaticExplain(pReq->pDag, (SRetrieveTableRsp **)&pJob->resData)); @@ -1571,7 +1632,7 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { if (NULL == schAcquireJob(refId)) { SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } pJob->refId = refId; @@ -1582,9 +1643,11 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { *job = pJob->refId; SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - - if (!pJob->attr.syncSchedule) { - code = schNotifyUserQueryRes(pJob); + + if (!sync) { + schPostJobRes(pJob, SCH_OP_EXEC); + } else { + schEndOperation(pJob); } schReleaseJob(pJob->refId); @@ -1593,56 +1656,29 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { _return: + schEndOperation(pJob); + if (!sync) { + pReq->fp(NULL, pReq->cbParam, code); + } + schFreeJobImpl(pJob); + SCH_RET(code); } int32_t schFetchRows(SSchJob *pJob) { int32_t code = 0; - int8_t status = SCH_GET_JOB_STATUS(pJob); - if (status == JOB_TASK_STATUS_DROPPING) { - SCH_JOB_ELOG("job is dropping, status:%s", jobTaskStatusStr(status)); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); - } - - if (!SCH_JOB_NEED_FETCH(pJob)) { - SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { - SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); - SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); - } else if (status == JOB_TASK_STATUS_SUCCEED) { - SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); - goto _return; - } else if (status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); - } - if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { SCH_ERR_JRET(schFetchFromRemote(pJob)); tsem_wait(&pJob->rspSem); - - status = SCH_GET_JOB_STATUS(pJob); - if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); - SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); - } } SCH_ERR_JRET(schSetJobFetchRes(pJob, pJob->userRes.fetchRes)); _return: - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); + schEndOperation(pJob); SCH_RET(code); } @@ -1650,50 +1686,14 @@ _return: int32_t schAsyncFetchRows(SSchJob *pJob) { int32_t code = 0; - int8_t status = SCH_GET_JOB_STATUS(pJob); - if (status == JOB_TASK_STATUS_DROPPING) { - SCH_JOB_ELOG("job is dropping, status:%s", jobTaskStatusStr(status)); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); - } - - if (!SCH_JOB_NEED_FETCH(pJob)) { - SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - if (atomic_val_compare_exchange_8(&pJob->userFetch, 0, 1) != 0) { - SCH_JOB_ELOG("prior fetching not finished, userFetch:%d", atomic_load_8(&pJob->userFetch)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - if (JOB_TASK_STATUS_FAILED == status || JOB_TASK_STATUS_DROPPING == status) { - SCH_JOB_ELOG("job failed or dropping, status:%s", jobTaskStatusStr(status)); - SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); - } else if (status == JOB_TASK_STATUS_SUCCEED) { - SCH_JOB_DLOG("job already succeed, status:%s", jobTaskStatusStr(status)); - goto _return; - } else if (status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); - } - if (pJob->attr.explainMode == EXPLAIN_MODE_STATIC) { - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - - SCH_ERR_JRET(schNotifyUserFetchRes(pJob)); - } else { - pJob->userCb = SCH_FETCH_CB; - - SCH_ERR_JRET(schFetchFromRemote(pJob)); + schPostJobRes(pJob, SCH_OP_FETCH); + return TSDB_CODE_SUCCESS; } + + SCH_ERR_RET(schFetchFromRemote(pJob)); return TSDB_CODE_SUCCESS; - -_return: - - atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - - SCH_RET(code); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index ddcb53d62f..2e9e5e5d1a 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -315,8 +315,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch return TSDB_CODE_SUCCESS; } - atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); - SCH_ERR_JRET(schFetchFromRemote(pJob)); taosMemoryFreeClear(msg); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 66483187da..93e1fda1d3 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -21,6 +21,18 @@ #include "tref.h" #include "trpc.h" +char* schGetOpStr(SCH_OP_TYPE type) { + switch (type) { + case SCH_OP_NULL: + return "NULL"; + case SCH_OP_EXEC: + return "EXEC"; + case SCH_OP_FETCH: + return "FETCH"; + default: + return "UNKNOWN"; + } +} void schCleanClusterHb(void* pTrans) { SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 0eaeeae9cb..1f2780e1d7 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -78,16 +78,18 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) { int32_t code = 0; if (NULL == pReq || NULL == pJob) { - code = TSDB_CODE_QRY_INVALID_INPUT; - } else { - code = schAsyncExecJob(pReq, pJob); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + + schAsyncExecJob(pReq, pJob); + +_return: if (code != TSDB_CODE_SUCCESS) { pReq->fp(NULL, pReq->cbParam, code); } - return code; + SCH_RET(code); } int32_t schedulerFetchRows(int64_t job, void **pData) { @@ -102,7 +104,8 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - pJob->attr.syncSchedule = true; + SCH_ERR_RET(schBeginOperation(pJob, SCH_OP_FETCH, true)); + pJob->userRes.fetchRes = pData; code = schFetchRows(pJob); @@ -112,24 +115,30 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { } void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) { + int32_t code = 0; if (NULL == fp || NULL == param) { - fp(NULL, param, TSDB_CODE_QRY_INVALID_INPUT); - return; + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { - qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job); - fp(NULL, param, TSDB_CODE_SCH_STATUS_ERROR); - return; + qError("acquire sch job from job list failed, may be dropped, jobId:0x%" PRIx64, job); + SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - pJob->attr.syncSchedule = false; + SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_FETCH, false)); + pJob->userRes.fetchFp = fp; pJob->userRes.userParam = param; - /*code = */schAsyncFetchRows(pJob); + SCH_ERR_JRET(schAsyncFetchRows(pJob)); +_return: + + if (code) { + fp(NULL, param, code); + } + schReleaseJob(job); } @@ -188,15 +197,17 @@ void schedulerStopQueryHb(void *pTrans) { schCleanClusterHb(pTrans); } -void schedulerFreeJob(int64_t job) { +void schedulerFreeJob(int64_t job, int32_t errCode) { SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job); return; } - if (atomic_load_8(&pJob->userFetch) > 0) { - schProcessOnJobDropped(pJob, TSDB_CODE_QRY_JOB_FREED); + int32_t code = schProcessOnJobDropped(pJob, errCode); + if (TSDB_CODE_SCH_JOB_IS_DROPPING == code) { + SCH_JOB_DLOG("sch job is already dropping, refId:%" PRIx64, job); + return; } SCH_JOB_DLOG("start to remove job from jobRef list, refId:%" PRIx64, job); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 9b624ee5cd..e5cc3cd481 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -457,7 +457,7 @@ void schtFreeQueryJob(int32_t freeThread) { int64_t job = queryJobRefId; if (job && atomic_val_compare_exchange_64(&queryJobRefId, job, 0)) { - schedulerFreeJob(job); + schedulerFreeJob(job, 0); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { printf("FreeNum:%d\n", freeNum); @@ -724,7 +724,7 @@ TEST(queryTest, normalCase) { schReleaseJob(job); - schedulerFreeJob(job); + schedulerFreeJob(job, 0); schtFreeQueryDag(&dag); @@ -828,7 +828,7 @@ TEST(queryTest, readyFirstCase) { schReleaseJob(job); - schedulerFreeJob(job); + schedulerFreeJob(job, 0); schtFreeQueryDag(&dag); @@ -940,7 +940,7 @@ TEST(queryTest, flowCtrlCase) { schReleaseJob(job); - schedulerFreeJob(job); + schedulerFreeJob(job, 0); schtFreeQueryDag(&dag); @@ -994,7 +994,7 @@ TEST(insertTest, normalCase) { ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); - schedulerFreeJob(insertJobRefId); + schedulerFreeJob(insertJobRefId, 0); schedulerDestroy(); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index fe040df6fc..07098d3041 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -132,6 +132,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_API_ERROR, "Stmt API usage error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not set") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") @@ -450,6 +451,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_VG_META_MISMATCH, "table meta and vgroup TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_TIMEOUT_ERROR, "Task timeout") +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_JOB_IS_DROPPING, "Job is dropping") TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") // parser diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index 95201bc48e..f23d75d78b 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -98,7 +98,7 @@ if $rows != 1 then endi #sql select * from information_schema.`streams` sql select * from information_schema.user_tables -if $rows != 28 then +if $rows != 29 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -196,7 +196,7 @@ if $rows != 1 then endi #sql select * from performance_schema.`streams` sql select * from information_schema.user_tables -if $rows != 28 then +if $rows != 29 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -210,4 +210,4 @@ if $rows != 3 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode2 -s stop -x SIGINT From cc638a2236ad0b802443e198c467dc41fff30d05 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 Jun 2022 21:13:18 +0800 Subject: [PATCH 63/90] fix(query): set null for null value in groupby --- source/libs/executor/src/executorimpl.c | 3 +-- source/libs/executor/src/groupoperator.c | 1 - source/libs/function/src/builtinsimpl.c | 19 +++++++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 66849e49b8..40b019eb5d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3507,8 +3507,7 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) // check for the limitation in each group if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); - - if (pProjectInfo->slimit.limit != -1 && pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { + if (pProjectInfo->slimit.limit > 0 && pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { pOperator->status = OP_EXEC_DONE; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 23226a0134..4fc25688c4 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -331,7 +331,6 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { } pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pInfo->binfo.resultRowInfo); #if 0 if(pOperator->fpSet.encodeResultRow){ diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 90700580bd..55900c7c7e 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1384,8 +1384,9 @@ int32_t maxFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, - int32_t rowIndex); +static void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); + +static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rIndex); int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); @@ -1407,11 +1408,23 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pEntryInfo->numOfRes > 0) { setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); + } else { + setNullSelectivityValue(pCtx, pBlock, currentRow); } return pEntryInfo->numOfRes; } +void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex) { + for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { + SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; + int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + colDataAppendNULL(pDstCol, rowIndex); + } +} + void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; @@ -4627,8 +4640,6 @@ int32_t tailFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STailItem* pItem = pInfo->pItems[i]; colDataAppend(pCol, currentRow, pItem->data, false); - - // setSelectivityValue(pCtx, pBlock, &pInfo->pItems[i].tuplePos, currentRow); currentRow += 1; } From ae460311214631736127b6951be772f53674a3d5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Jun 2022 21:31:45 +0800 Subject: [PATCH 64/90] fix: mnode basic5.sim --- source/dnode/mnode/impl/src/mndMain.c | 1 + source/dnode/mnode/impl/src/mndMnode.c | 2 +- tests/script/tsim/mnode/basic5.sim | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index f277a5169d..7b5025a986 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -505,6 +505,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { } static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { + if (!IsReq(pMsg)) return 0; if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; if (IsReq(pMsg) && pMsg->msgType != TDMT_MND_MQ_TIMER && pMsg->msgType != TDMT_MND_TELEM_TIMER && diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index e82c479cdd..a636f34f3d 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -742,7 +742,7 @@ static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq) { pMgmt->errCode = 0; pMgmt->transId = -1; tsem_wait(&pMgmt->syncSem); - mInfo("alter mnode sync result:%s", tstrerror(pMgmt->errCode)); + mInfo("alter mnode sync result:0x%x %s", pMgmt->errCode, tstrerror(pMgmt->errCode)); terrno = pMgmt->errCode; return pMgmt->errCode; } diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index 399dced169..fc591aa25d 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -3,6 +3,10 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 system sh/exec.sh -n dnode1 -s start sql connect @@ -260,7 +264,7 @@ $x = 0 step92: $x = $x + 1 sleep 1000 - if $x == 10 then + if $x == 20 then return -1 endi sql show mnodes From 79494364aecd2acecf5953b41e53f90b69dc5985 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 16 Jun 2022 22:00:33 +0800 Subject: [PATCH 65/90] refactor(tmq): offset storage --- examples/c/tmq.c | 7 ++- include/client/taos.h | 90 +++++++++++++--------------- source/client/src/tmq.c | 62 +++++++++++++------ source/dnode/vnode/src/tq/tq.c | 26 +++++--- source/dnode/vnode/src/tq/tqCommit.c | 5 +- source/dnode/vnode/src/tq/tqOffset.c | 4 +- tests/test/c/tmqSim.c | 70 ++++++++++------------ 7 files changed, 144 insertions(+), 120 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 2e8aa21da7..6b09e8c658 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -47,7 +47,7 @@ int32_t init_env() { return -1; } - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); if (taos_errno(pRes) != 0) { printf("error in create db, reason:%s\n", taos_errstr(pRes)); return -1; @@ -167,7 +167,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); - tmq_conf_set(conf, "enable.auto.commit", "false"); + tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); @@ -201,6 +201,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { taos_free_result(tmqmessage); /*} else {*/ /*break;*/ + /*tmq_commit_sync(tmq, NULL);*/ } } @@ -239,7 +240,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { msg_process(tmqmessage); taos_free_result(tmqmessage); - /*tmq_commit_async(tmq, NULL, tmq_commit_cb_print, NULL);*/ + /*tmq_commit_sync(tmq, NULL);*/ /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ } } diff --git a/include/client/taos.h b/include/client/taos.h index deb4276b54..1510f38074 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -126,47 +126,47 @@ typedef struct setConfRet { char retMsg[RET_MSG_LENGTH]; } setConfRet; -DLL_EXPORT void taos_cleanup(void); -DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); -DLL_EXPORT setConfRet taos_set_config(const char *config); -DLL_EXPORT int taos_init(void); -DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); +DLL_EXPORT void taos_cleanup(void); +DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT setConfRet taos_set_config(const char *config); +DLL_EXPORT int taos_init(void); +DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); -const char *taos_data_type(int type); +const char *taos_data_type(int type); -DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); -DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); +DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); @@ -181,8 +181,8 @@ DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnInde DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT void taos_reset_current_db(TAOS *taos); -DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); -DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); +DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); @@ -192,8 +192,8 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); -DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES* res, __taos_async_fn_t fp, void* param); -DLL_EXPORT const void *taos_get_raw_block(TAOS_RES* res); +DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); +DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); // Shuduo: temporary enable for app build #if 1 @@ -240,11 +240,10 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); - // timeout: -1 means infinitely waiting -DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); - +DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); + DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); @@ -275,11 +274,6 @@ DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); -#if 0 -DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message); -DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message); -#endif - /* ------------------------------ TMQ END -------------------------------- */ #if 1 // Shuduo: temporary enable for app build diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index dd900e6045..38b1872206 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -203,10 +203,10 @@ typedef struct { int32_t totalRspNum; tmq_resp_err_t rspErr; tmq_commit_cb* userCb; - SArray* successfulOffsets; - SArray* failedOffsets; - void* userParam; - tsem_t rspSem; + /*SArray* successfulOffsets;*/ + /*SArray* failedOffsets;*/ + void* userParam; + tsem_t rspSem; } SMqCommitCbParamSet; typedef struct { @@ -368,11 +368,13 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; // push into array +#if 0 if (code == 0) { taosArrayPush(pParamSet->failedOffsets, &pParam->pOffset); } else { taosArrayPush(pParamSet->successfulOffsets, &pParam->pOffset); } +#endif // count down waiting rsp int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); @@ -388,10 +390,14 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { // sem post pParamSet->userCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->userParam); } + } else { + tsem_post(&pParamSet->rspSem); } +#if 0 taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); +#endif } return 0; } @@ -417,11 +423,17 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, i); - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + if (pVg->currentOffset < 0) { + /*if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {*/ + continue; + } + STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); if (pOffset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pOffset->type = TMQ_OFFSET__LOG; + pOffset->version = pVg->currentOffset; int32_t tlen = strlen(tmq->groupId); memcpy(pOffset->subKey, tmq->groupId, tlen); pOffset->subKey[tlen] = TMQ_SEPARATOR; @@ -454,25 +466,33 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 } pMsgSendInfo->msgInfo = (SDataBuf){ .pData = buf, - .len = len, + .len = sizeof(SMsgHead) + len, .handle = NULL, }; + // TODO: put into cb + pVg->committedOffset = pVg->currentOffset; + pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; pMsgSendInfo->fp = tmqCommitCb2; - pMsgSendInfo->msgType = TDMT_MND_MQ_COMMIT_OFFSET; + pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); pParamSet->waitingRspNum++; pParamSet->totalRspNum++; } } + if (pParamSet->totalRspNum == 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); + return 0; + } + if (!async) { tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; @@ -489,10 +509,12 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 } } +#if 0 if (!async) { taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); } +#endif return 0; } @@ -648,7 +670,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { tmqAskEp(tmq, true); taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); + tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else { @@ -819,7 +841,7 @@ FAIL: } tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { - return tmqCommitInner(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); + return tmqCommitInner2(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); } tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { @@ -1076,6 +1098,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } +#if 0 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ bool set = false; @@ -1160,6 +1183,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { atomic_store_32(&tmq->epoch, epoch); return set; } +#endif int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; @@ -1186,7 +1210,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ - tmqUpdateEp(tmq, head->epoch, &rsp); + tmqUpdateEp2(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); @@ -1311,10 +1335,10 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* if (pVg->currentOffset >= 0) { reqOffset = pVg->currentOffset; } else { - if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) { - tscError("unable to poll since no committed offset but reset offset is set to none"); - return NULL; - } + /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ + /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ + /*return NULL;*/ + /*}*/ reqOffset = tmq->resetOffsetCfg; } @@ -1440,7 +1464,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); + tmqUpdateEp2(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; } else { @@ -1608,9 +1632,9 @@ const char* tmq_get_table_name(TAOS_RES* res) { } void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) { - tmqCommitInner(tmq, offsets, 0, 1, cb, param); + tmqCommitInner2(tmq, offsets, 0, 1, cb, param); } tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - return tmqCommitInner(tmq, offsets, 0, 0, NULL, NULL); + return tmqCommitInner2(tmq, offsets, 0, 0, NULL, NULL); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c9d3267adb..a7b3fc8a2f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -149,16 +149,28 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { int32_t code = 0; // get offset to fetch message - if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = walGetFirstVer(pTq->pWal); - } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { - fetchOffset = walGetCommittedVer(pTq->pWal); - } else { + if (pReq->currentOffset >= 0) { fetchOffset = pReq->currentOffset + 1; + } else { + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); + if (pOffset != NULL) { + ASSERT(pOffset->type == TMQ_OFFSET__LOG); + fetchOffset = pOffset->version + 1; + } else { + if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { + fetchOffset = walGetFirstVer(pTq->pWal); + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { + fetchOffset = walGetCommittedVer(pTq->pWal); + } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__NONE) { + tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s", consumerId, + pTq->pVnode->config.vgId, pReq->subKey); + return -1; + } + } } - tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %ld fetch offset %ld", consumerId, + pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); /*ASSERT(pHandle);*/ diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index 7b116bff2e..639da22b1c 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -15,7 +15,4 @@ #include "tq.h" -int tqCommit(STQ* pTq) { - // do nothing - return 0; -} +int tqCommit(STQ* pTq) { return tqOffsetSnapshot(pTq->pOffsetStore); } diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 8d6cb28065..e5475d7c30 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -92,6 +92,8 @@ STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) { } int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { + ASSERT(pOffset->type == TMQ_OFFSET__LOG); + ASSERT(pOffset->version >= 0); return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)); } @@ -129,7 +131,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { tEncodeSTqOffset(&encoder, pOffset); // write file int64_t writeLen; - if ((writeLen = taosWriteFile(pFile, buf, totLen)) != bodyLen) { + if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { ASSERT(0); tqError("write offset incomplete, len %d, write len %ld", bodyLen, writeLen); taosHashCancelIterate(pStore->pHash, pIter); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 8455bd9890..8c2a660dd6 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -62,7 +62,7 @@ typedef struct { tmq_t* tmq; tmq_list_t* topicList; - + int32_t numOfVgroups; int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume int64_t ts; @@ -74,7 +74,7 @@ typedef struct { char cdbName[32]; char dbName[32]; int32_t showMsgFlag; - int32_t showRowFlag; + int32_t showRowFlag; int32_t saveRowFlag; int32_t consumeDelay; // unit s int32_t numOfThread; @@ -108,26 +108,20 @@ static void printHelp() { } char* getCurrentTimeString(char* timeString) { - time_t tTime = taosGetTimestampSec(); + time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); - sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", - tm.tm_year + 1900, - tm.tm_mon + 1, - tm.tm_mday, - tm.tm_hour, - tm.tm_min, - tm.tm_sec); + sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec); return timeString; } - void initLogFile() { char filename[256]; - char tmpString[128]; + char tmpString[128]; - sprintf(filename,"%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString)); - //sprintf(filename, "%s/../log/tmqlog.txt", configDir); + sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString)); + // sprintf(filename, "%s/../log/tmqlog.txt", configDir); #ifdef WINDOWS for (int i = 2; i < sizeof(filename); i++) { if (filename[i] == ':') filename[i] = '-'; @@ -249,17 +243,18 @@ void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { for (i = 0; i < pInfo->numOfVgroups; i++) { if (vgroupId == pInfo->rowsOfPerVgroups[i][0]) { pInfo->rowsOfPerVgroups[i][1] += rows; - return; - } + return; + } } pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][0] = vgroupId; pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows; pInfo->numOfVgroups++; - + taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId); if (pInfo->numOfVgroups > MAX_VGROUP_CNT) { - taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, pInfo->numOfVgroups, vgroupId); + taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, + pInfo->numOfVgroups, vgroupId); taosCloseFile(&g_fp); exit(-1); } @@ -277,7 +272,8 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - sprintf(sqlStr, "insert into %s.content_%d values (%"PRId64", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->ts++, buf); + sprintf(sqlStr, "insert into %s.content_%d values (%" PRId64 ", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, + pInfo->ts++, buf); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { pError("error in insert consume result, reason:%s\n", taos_errstr(pRes)); @@ -295,12 +291,13 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { char buf[1024]; int32_t totalRows = 0; - + // printf("topic: %s\n", tmq_get_topic_name(msg)); int32_t vgroupId = tmq_get_vgroup_id(msg); - + taosFprintfFile(g_fp, "msg index:%" PRId64 ", consumerId: %d\n", msgIndex, pInfo->consumerId); - //taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId, tmq_get_table_name(msg)); + // taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId, + // tmq_get_table_name(msg)); taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), vgroupId); while (1) { @@ -316,9 +313,9 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) const char* tbName = tmq_get_table_name(msg); if (0 != g_stConfInfo.showRowFlag) { - taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName:"null table"), totalRows, buf); - if (0 != g_stConfInfo.saveRowFlag) { - saveConsumeContentToTbl(pInfo, buf); + taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); + if (0 != g_stConfInfo.saveRowFlag) { + saveConsumeContentToTbl(pInfo, buf); } } @@ -326,7 +323,7 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) } addRowsToVgroupId(pInfo, vgroupId, totalRows); - + return totalRows; } @@ -401,16 +398,11 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { int64_t now = taosGetTimestampMs(); // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (%"PRId64", %d, %" PRId64 ", %" PRId64 ", %d)", - g_stConfInfo.cdbName, - now, - pInfo->consumerId, - pInfo->consumeMsgCnt, - pInfo->consumeRowCnt, - pInfo->checkresult); + sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, now, pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult); char tmpString[128]; - taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId ,sqlStr); + taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { @@ -421,7 +413,7 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { taos_free_result(pRes); - #if 0 +#if 0 // vgroups for (i = 0; i < pInfo->numOfVgroups; i++) { // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int @@ -445,7 +437,7 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { taos_free_result(pRes); } - #endif +#endif return 0; } @@ -457,7 +449,8 @@ void loop_consume(SThreadInfo* pInfo) { int64_t totalRows = 0; char tmpString[128]; - taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), pInfo->consumerId); + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); pInfo->ts = taosGetTimestampMs(); @@ -473,7 +466,7 @@ void loop_consume(SThreadInfo* pInfo) { totalMsgs++; if (totalRows >= pInfo->expectMsgCnt) { - char tmpString[128]; + char tmpString[128]; taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString)); break; } @@ -519,6 +512,7 @@ void* consumeThreadFunc(void* param) { pPrint("tmq_commit() manual commit when consume end.\n"); /*tmq_commit(pInfo->tmq, NULL, 0);*/ tmq_commit_sync(pInfo->tmq, NULL); + taosFprintfFile(g_fp, "tmq_commit() manual commit over.\n"); } err = tmq_unsubscribe(pInfo->tmq); From 50c422bf7417d7031313622764d1a7087ac059c1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 08:25:30 +0800 Subject: [PATCH 66/90] fix: dnode removed if create vnode failed --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/libs/tfs/src/tfs.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 9946185da6..f112727c08 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -228,7 +228,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; code = terrno; - goto _OVER; + return 0; } snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 92beeffa0c..2d3d41de64 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -282,6 +282,8 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) { } int32_t tfsRmdir(STfs *pTfs, const char *rname) { + ASSERT(rname[0] != 0); + char aname[TMPNAME_LEN] = "\0"; for (int32_t level = 0; level < pTfs->nlevel; level++) { @@ -289,6 +291,7 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname) { for (int32_t id = 0; id < pTier->ndisk; id++) { STfsDisk *pDisk = pTier->disks[id]; snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); + uInfo("====> tfs remove dir : path:%s aname:%s rname:[%s]", pDisk->path, aname, rname); taosRemoveDir(aname); } } From feebe90a1f27b4c9be0020cb5049a5949bd122a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Jun 2022 09:18:53 +0800 Subject: [PATCH 67/90] fix(query): set correct buffer page size. --- source/libs/executor/src/scanoperator.c | 50 +++++++++++-------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ddebba1c9e..8306c0bcc4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2170,25 +2170,17 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, int32_t capa blockDataEnsureCapacity(p, capacity); while (1) { - STupleHandle* pTupleHandle = NULL; - if (pInfo->prefetchedTuple == NULL) { - pTupleHandle = tsortNextTuple(pHandle); - } else { - pTupleHandle = pInfo->prefetchedTuple; - } - + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); if (pTupleHandle == NULL) { break; } appendOneRowToDataBlock(p, pTupleHandle); - if (p->info.rows >= capacity) { break; } } - qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), p->info.rows); return (p->info.rows > 0) ? p : NULL; } @@ -2261,7 +2253,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -2284,17 +2275,16 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; - pInfo->readHandle = *readHandle; - pInfo->interval = extractIntervalInfo(pTableScanNode); + pInfo->readHandle = *readHandle; + pInfo->interval = extractIntervalInfo(pTableScanNode); pInfo->sample.sampleRatio = pTableScanNode->ratio; - pInfo->sample.seed = taosGetTimestampSec(); - - pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; - pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; - pInfo->dataReaders = dataReaders; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColList; - pInfo->curTWinIdx = 0; + pInfo->sample.seed = taosGetTimestampSec(); + pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; + pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + pInfo->dataReaders = dataReaders; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColList; + pInfo->curTWinIdx = 0; pInfo->pResBlock = createResDataBlock(pDescNode); @@ -2307,22 +2297,24 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN taosArrayPush(pInfo->sortSourceParams, param); taosMemoryFree(param); } + pInfo->pSortInfo = generateSortByTsInfo(pInfo->cond.order); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); + int32_t rowSize = pInfo->pResBlock->info.rowSize; - pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2; - pInfo->sortBufSize = pInfo->bufPageSize * 16; - pInfo->hasGroupId = false; + pInfo->bufPageSize = getProperSortPageSize(rowSize); + pInfo->sortBufSize = pInfo->bufPageSize * 16; + pInfo->hasGroupId = false; pInfo->prefetchedTuple = NULL; - pOperator->name = "TableMergeScanOperator"; + pOperator->name = "TableMergeScanOperator"; // TODO : change it pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = numOfCols; + pOperator->pTaskInfo = pTaskInfo; initResultSizeInfo(pOperator, 1024); pOperator->fpSet = From af7f3e3252512eae37bc1684dfaab821b9b7c5a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Jun 2022 09:20:19 +0800 Subject: [PATCH 68/90] refactor(query): set correct buffer pages. --- source/libs/executor/src/scanoperator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8306c0bcc4..69fa856215 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2303,7 +2303,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - pInfo->sortBufSize = pInfo->bufPageSize * 16; + + // the additional one is reserved for merge result + pInfo->sortBufSize = pInfo->bufPageSize * (taosArrayGetSize(dataReaders) + 1); pInfo->hasGroupId = false; pInfo->prefetchedTuple = NULL; From d12614aac8f7e0734d0d2602fd20fbe4d0f78c42 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Jun 2022 09:22:03 +0800 Subject: [PATCH 69/90] kill query --- include/libs/scheduler/scheduler.h | 13 ++-- source/client/src/clientImpl.c | 6 +- source/libs/scheduler/inc/schedulerInt.h | 1 + source/libs/scheduler/src/schJob.c | 84 ++++++++++++++---------- 4 files changed, 60 insertions(+), 44 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index dfff17221d..ecb21335b9 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -73,13 +73,14 @@ typedef void (*schedulerExecCallback)(SQueryResult* pResult, void* param, int32_ typedef void (*schedulerFetchCallback)(void* pResult, void* param, int32_t code); typedef struct SSchedulerReq { - SRequestConnInfo *pConn; - SArray *pNodeList; - SQueryPlan *pDag; - const char *sql; - int64_t startTs; + bool *reqKilled; + SRequestConnInfo *pConn; + SArray *pNodeList; + SQueryPlan *pDag; + const char *sql; + int64_t startTs; schedulerExecCallback fp; - void* cbParam; + void* cbParam; } SSchedulerReq; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a9dd1bb50b..6f0f1c7f41 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -461,7 +461,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, .fp = NULL, - .cbParam = NULL}; + .cbParam = NULL, + .reqKilled = &pRequest->killed}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; @@ -738,7 +739,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, .fp = schedulerExecCb, - .cbParam = pRequest}; + .cbParam = pRequest, + .reqKilled = &pRequest->killed}; code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); } else { tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 2a3a069d02..545bb6c45a 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -228,6 +228,7 @@ typedef struct SSchJob { SQueryNodeAddr resNode; tsem_t rspSem; SSchOpStatus opStatus; + bool *reqKilled; SSchTask *fetchTask; int32_t errCode; SRWLatch resLock; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index d723a431c2..f9c7b21cf2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -54,6 +54,7 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; pJob->sql = pReq->sql; + pJob->reqKilled = pReq->reqKilled; pJob->userRes.queryRes = pRes; pJob->userRes.execFp = pReq->fp; pJob->userRes.userParam = pReq->cbParam; @@ -154,12 +155,52 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) { } } + +void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { + if (TSDB_CODE_SUCCESS == errCode) { + return; + } + + int32_t origCode = atomic_load_32(&pJob->errCode); + if (TSDB_CODE_SUCCESS == origCode) { + if (origCode == atomic_val_compare_exchange_32(&pJob->errCode, origCode, errCode)) { + goto _return; + } + + origCode = atomic_load_32(&pJob->errCode); + } + + if (NEED_CLIENT_HANDLE_ERROR(origCode)) { + return; + } + + if (NEED_CLIENT_HANDLE_ERROR(errCode)) { + atomic_store_32(&pJob->errCode, errCode); + goto _return; + } + + return; + +_return: + + SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); +} + + + FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { int8_t status = SCH_GET_JOB_STATUS(pJob); if (pStatus) { *pStatus = status; } + if (pJob->reqKilled) { + schUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); + schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED); + + return true; + } + return (status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_DROPPING || status == JOB_TASK_STATUS_SUCCEED); } @@ -255,7 +296,13 @@ void schEndOperation(SSchJob *pJob) { int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync) { int32_t code = 0; + int8_t status = 0; + if (schJobNeedToStop(pJob, &status)) { + SCH_JOB_ELOG("job need to stop cause of status %s", jobTaskStatusStr(status)); + SCH_ERR_JRET(pJob->errCode); + } + if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_ERR_JRET(TSDB_CODE_TSC_APP_ERROR); @@ -275,11 +322,7 @@ int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - int8_t status = 0; - if (schJobNeedToStop(pJob, &status)) { - SCH_JOB_ELOG("job need to stop cause of status %s", jobTaskStatusStr(status)); - SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); - } else if (status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { + if (status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_JOB_ELOG("job status error for fetch, status:%s", jobTaskStatusStr(status)); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -841,37 +884,6 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { - if (TSDB_CODE_SUCCESS == errCode) { - return; - } - - int32_t origCode = atomic_load_32(&pJob->errCode); - if (TSDB_CODE_SUCCESS == origCode) { - if (origCode == atomic_val_compare_exchange_32(&pJob->errCode, origCode, errCode)) { - goto _return; - } - - origCode = atomic_load_32(&pJob->errCode); - } - - if (NEED_CLIENT_HANDLE_ERROR(origCode)) { - return; - } - - if (NEED_CLIENT_HANDLE_ERROR(errCode)) { - atomic_store_32(&pJob->errCode, errCode); - goto _return; - } - - return; - -_return: - - SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); -} - - int32_t schSetJobQueryRes(SSchJob* pJob, SQueryResult* pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; From a4a8d38ca26c0728f1b6165f1f50902e57305c83 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 09:25:15 +0800 Subject: [PATCH 70/90] fix: minor changes --- source/dnode/mnode/impl/src/mndSync.c | 5 +++-- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 +- source/util/src/terror.c | 7 ++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 38b75db027..ee5a291fbb 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -59,8 +59,8 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s", transId, tstrerror(pMgmt->errCode)); } - tsem_post(&pMgmt->syncSem); pMgmt->transId = 0; + tsem_post(&pMgmt->syncSem); } else { STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { @@ -123,8 +123,8 @@ void mndReConfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbM if (pMgmt->errCode != 0) { mError("trans:-1, failed to propose sync reconfig since %s", tstrerror(pMgmt->errCode)); } - tsem_post(&pMgmt->syncSem); pMgmt->transId = 0; + tsem_post(&pMgmt->syncSem); } } @@ -262,6 +262,7 @@ void mndSyncStart(SMnode *pMnode) { void mndSyncStop(SMnode *pMnode) { if (pMnode->syncMgmt.transId != 0) { + pMnode->syncMgmt.transId = 0; tsem_post(&pMnode->syncMgmt.syncSem); } } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 874210306f..3715866bb8 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -220,7 +220,7 @@ int vnodeCommit(SVnode *pVnode) { info.state.committed = pVnode->state.applied; snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); if (vnodeSaveInfo(dir, &info) < 0) { - // ASSERT(0); + ASSERT(0); return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 596aab0e09..933e3ba92b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -276,9 +276,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY, "Topic with invalid query") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Topic with invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Consumer unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") +// mnode-stream TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_NOT_EXIST, "Stream not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option") From 793e6fa7d9f365aed85c1854a4cb7fc7789a01f8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 09:26:25 +0800 Subject: [PATCH 71/90] fix: rollback to avoid tsim/basic.sim failure --- source/libs/sync/src/syncRaftLog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 7c6d97a96e..26da607c8e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -410,7 +410,7 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { } int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { - /* + SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; // assert(walCommit(pWal, index) == 0); @@ -423,7 +423,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); ASSERT(0); } - */ + return 0; } From 5407ea7895a306c5de59af002275d6494e3b24c4 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 17 Jun 2022 09:32:32 +0800 Subject: [PATCH 72/90] docs: reference rust --- docs/en/14-reference/03-connector/rust.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/14-reference/03-connector/rust.mdx b/docs/en/14-reference/03-connector/rust.mdx index a5cbaeac80..56ca586c7e 100644 --- a/docs/en/14-reference/03-connector/rust.mdx +++ b/docs/en/14-reference/03-connector/rust.mdx @@ -250,7 +250,7 @@ The [Taos] structure is the connection manager in [libtaos] and provides two mai Column information is stored using [ColumnMeta]. - ``rust + ```rust let cols = &q.column_meta; for col in cols { println!("name: {}, type: {:?} , bytes: {}", col.name, col.type_, col.bytes); From 80e0b830fe09470d2bd75bc24809723fbabfb0db Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 10:40:42 +0800 Subject: [PATCH 73/90] refactor: remove duplicate conninfo in SRpcMsg --- include/libs/transport/trpc.h | 5 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 25 +------ source/dnode/mnode/impl/src/mndAuth.c | 2 +- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 12 ++-- source/dnode/mnode/impl/src/mndDnode.c | 4 +- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndProfile.c | 67 ++++++++++--------- source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 6 +- source/dnode/mnode/impl/src/mndStream.c | 8 +-- source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 2 +- source/dnode/mnode/impl/src/mndUser.c | 8 +-- source/dnode/mnode/impl/src/mndVgroup.c | 6 +- source/dnode/vnode/src/vnd/vnodeSync.c | 6 +- source/libs/transport/src/transSvr.c | 2 +- 21 files changed, 82 insertions(+), 99 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index fd57eef83a..e3db1550a0 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -46,7 +46,6 @@ typedef struct SRpcHandleInfo { int32_t noResp; // has response or not(default 0, 0: resp, 1: no resp); int32_t persistHandle; // persist handle or not - SRpcConnInfo connInfo; // app info void *ahandle; // app handle set by client void *wrapper; // wrapper handle @@ -55,6 +54,9 @@ typedef struct SRpcHandleInfo { // resp info void * rsp; int32_t rspLen; + + // conn info + SRpcConnInfo conn; } SRpcHandleInfo; typedef struct SRpcMsg { @@ -63,7 +65,6 @@ typedef struct SRpcMsg { int32_t contLen; int32_t code; SRpcHandleInfo info; - SRpcConnInfo conn; } SRpcMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index b66e559370..13f2452e66 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -21,21 +21,6 @@ static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); static void dmSendRsp(SRpcMsg *pMsg); static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); -static inline int32_t dmBuildNodeMsg(SRpcMsg *pMsg, SRpcMsg *pRpc) { - SRpcConnInfo *pConnInfo = &(pRpc->info.connInfo); - // if (IsReq(pRpc)) { - // terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - // dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->info.ahandle, pRpc->info.handle); - // return -1; - //} - - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - memcpy(pMsg->conn.user, pConnInfo->user, TSDB_USER_LEN); - pMsg->conn.clientIp = pConnInfo->clientIp; - pMsg->conn.clientPort = pConnInfo->clientPort; - return 0; -} - int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; if (msgFp == NULL) { @@ -116,14 +101,10 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - if (pMsg == NULL) { - goto _OVER; - } - dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); + if (pMsg == NULL) goto _OVER; + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - if (dmBuildNodeMsg(pMsg, pRpc) != 0) { - goto _OVER; - } + dTrace("msg:%p, is created, type:%s handle:%p", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle); if (InParentProc(pWrapper)) { code = dmPutToProcCQueue(&pWrapper->proc, pMsg, DND_FUNC_REQ); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index f036fc48f7..d47fb9dfb4 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -56,7 +56,7 @@ static int32_t mndProcessAuthReq(SRpcMsg *pReq) { memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); int32_t code = mndRetriveAuth(pReq->info.node, &authRsp); - mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->conn.user, authRsp.spi, authRsp.encrypt, + mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->info.conn.user, authRsp.spi, authRsp.encrypt, authRsp.user); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index aa908b983d..0de40ca671 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -292,7 +292,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_BNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -394,7 +394,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_BNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c12c6d5b4c..2eeff9cb33 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -521,12 +521,12 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - pUser = mndAcquireUser(pMnode, pReq->conn.user); + pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { goto _OVER; } @@ -700,7 +700,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_ALTER_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -980,7 +980,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_DROP_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1127,7 +1127,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_USE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1252,7 +1252,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6d37143fc9..d1bed90175 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -522,7 +522,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_DNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { goto _OVER; } @@ -623,7 +623,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index dfdc0a3c1a..832f1b8e68 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -318,7 +318,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_FUNC) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -365,7 +365,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_FUNC) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a636f34f3d..b40cd713e5 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -414,7 +414,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_MNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { goto _OVER; } @@ -621,7 +621,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 20d94e341c..39d21aad49 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -133,7 +133,7 @@ static void mndFreeConn(SConnObj *pConn) { taosWLockLatch(&pConn->queryLock); taosArrayDestroyEx(pConn->pQueries, tFreeClientHbQueryDesc); taosWUnLockLatch(&pConn->queryLock); - + mTrace("conn:%u, is destroyed, data:%p", pConn->id, pConn); } @@ -194,15 +194,15 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto CONN_OVER; } - taosIp2String(pReq->conn.clientIp, ip); + taosIp2String(pReq->info.conn.clientIp, ip); - pUser = mndAcquireUser(pMnode, pReq->conn.user); + pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { - mError("user:%s, failed to login while acquire user since %s", pReq->conn.user, terrstr()); + mError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr()); goto CONN_OVER; } if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { - mError("user:%s, failed to auth while acquire user, input:%s", pReq->conn.user, connReq.passwd); + mError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd); code = TSDB_CODE_RPC_AUTH_FAILURE; goto CONN_OVER; } @@ -213,15 +213,16 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; - mError("user:%s, failed to login from %s while use db:%s since %s", pReq->conn.user, ip, connReq.db, terrstr()); + mError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db, + terrstr()); goto CONN_OVER; } } - pConn = mndCreateConn(pMnode, pReq->conn.user, connReq.connType, pReq->conn.clientIp, pReq->conn.clientPort, - connReq.pid, connReq.app, connReq.startTime); + pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp, + pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime); if (pConn == NULL) { - mError("user:%s, failed to login from %s while create connection since %s", pReq->conn.user, ip, terrstr()); + mError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr()); goto CONN_OVER; } @@ -246,7 +247,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pReq->info.rspLen = contLen; pReq->info.rsp = pRsp; - mDebug("user:%s, login from %s:%d, conn:%u, app:%s", pReq->conn.user, ip, pConn->port, pConn->id, connReq.app); + mDebug("user:%s, login from %s:%d, conn:%u, app:%s", pReq->info.conn.user, ip, pConn->port, pConn->id, connReq.app); code = 0; @@ -348,7 +349,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb if (pHbReq->query) { SQueryHbReqBasic *pBasic = pHbReq->query; - SRpcConnInfo connInfo = pMsg->conn; + SRpcConnInfo connInfo = pMsg->info.conn; SConnObj *pConn = mndAcquireConn(pMnode, pBasic->connId); if (pConn == NULL) { @@ -361,7 +362,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mDebug("user:%s, conn:%u is freed and create a new conn:%u", connInfo.user, pBasic->connId, pConn->id); } } - + SQueryHbRspBasic *rspBasic = taosMemoryCalloc(1, sizeof(SQueryHbRspBasic)); if (rspBasic == NULL) { mndReleaseConn(pMnode, pConn); @@ -386,7 +387,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb mndGetMnodeEpSet(pMnode, &rspBasic->epSet); mndCreateQnodeList(pMnode, &rspBasic->pQnodeList, -1); - + mndReleaseConn(pMnode, pConn); hbRsp.query = rspBasic; @@ -500,7 +501,7 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->conn.user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) return 0; if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); @@ -523,7 +524,7 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, queryId:%d is killed by user:%s", killReq.connId, killReq.queryId, pReq->conn.user); + mInfo("connId:%d, queryId:%d is killed by user:%s", killReq.connId, killReq.queryId, pReq->info.conn.user); pConn->killId = killReq.queryId; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return 0; @@ -534,7 +535,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->conn.user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) return 0; if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); @@ -555,7 +556,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, is killed by user:%s", killReq.connId, pReq->conn.user); + mInfo("connId:%d, is killed by user:%s", killReq.connId, pReq->info.conn.user); pConn->killed = 1; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; @@ -563,12 +564,12 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - int32_t cols = 0; - SConnObj *pConn = NULL; - + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SConnObj *pConn = NULL; + if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; pShow->pIter = taosCacheCreateIter(pMgmt->cache); @@ -619,12 +620,12 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - int32_t cols = 0; - SConnObj *pConn = NULL; - + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SConnObj *pConn = NULL; + if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; pShow->pIter = taosCacheCreateIter(pMgmt->cache); @@ -645,7 +646,7 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); for (int32_t i = 0; i < numOfQueries; ++i) { - SQueryDesc* pQuery = taosArrayGet(pConn->pQueries, i); + SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; char queryId[26 + VARSTR_HEADER_SIZE] = {0}; @@ -691,14 +692,14 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pQuery->subPlanNum, false); - char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; + char subStatus[TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t strSize = sizeof(subStatus); int32_t offset = VARSTR_HEADER_SIZE; for (int32_t i = 0; i < pQuery->subPlanNum && offset < strSize; ++i) { if (i) { offset += snprintf(subStatus + offset, strSize - offset - 1, ","); } - SQuerySubDesc* pDesc = taosArrayGet(pQuery->subDesc, i); + SQuerySubDesc *pDesc = taosArrayGet(pQuery->subDesc, i); offset += snprintf(subStatus + offset, strSize - offset - 1, "%" PRIu64 ":%s", pDesc->tid, pDesc->status); } varDataLen(subStatus) = strlen(&subStatus[VARSTR_HEADER_SIZE]); @@ -712,7 +713,7 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p numOfRows++; } - + taosRUnLockLatch(&pConn->queryLock); } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 595287a3af..f5625f32d5 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -294,7 +294,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_QNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -396,7 +396,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_QNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index d312955202..a6447ed405 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -229,7 +229,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - // if (mndCheckShowAuth(pMnode, pReq->conn.user, pShow->type) != 0) return -1; + // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 3eb5b03efd..023a28ce35 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -710,7 +710,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -971,7 +971,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index c84dc2f3dd..0a99f356b1 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -300,7 +300,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_SNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -404,7 +404,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_SNODE) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index b83aa34a3a..b8b22cee85 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -805,7 +805,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1454,7 +1454,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1584,7 +1584,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3a42c694c8..8e82946d68 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -545,7 +545,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SRpcMsg *pReq, SCMCreateStreamReq return -1; } - if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->conn.user) < 0) { + if (streamObj.targetSTbName[0] && mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mError("trans:%d, failed to create stb for stream since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -602,7 +602,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } #endif @@ -627,7 +627,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } // create stb for stream - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->conn.user) < 0) { + if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -696,7 +696,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { #if 0 // todo check auth - pUser = mndAcquireUser(pMnode, pReq->conn.user); + pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { goto DROP_STREAM_OVER; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 4c2730ce94..8afb7ab354 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -474,7 +474,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index dafb2fa1d1..b3a2888535 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1369,7 +1369,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { mInfo("trans:%d, start to kill", killReq.transId); - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_KILL_TRANS) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 9590823106..eb0a818a60 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -354,13 +354,13 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - pOperUser = mndAcquireUser(pMnode, pReq->conn.user); + pOperUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_CREATE_USER) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { goto _OVER; } @@ -460,7 +460,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - pOperUser = mndAcquireUser(pMnode, pReq->conn.user); + pOperUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; goto _OVER; @@ -643,7 +643,7 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_DROP_USER) != 0) { + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 755f4ef0b2..cec83d1af5 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1200,7 +1200,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { mInfo("vgId:%d, start to redistribute to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER; pVgroup = mndAcquireVgroup(pMnode, req.vgId); if (pVgroup == NULL) goto _OVER; @@ -1500,7 +1500,7 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, pVgroup->dbName); if (pDb == NULL) goto _OVER; - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) goto _OVER; code = mndSplitVgroup(pMnode, pReq, pDb, pVgroup); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -1624,7 +1624,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { mDebug("start to balance vgroup"); - if (mndCheckOperAuth(pMnode, pReq->conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER; + if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER; while (1) { SDnodeObj *pDnode = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 8662c8369b..6a1f5c606d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -184,7 +184,7 @@ void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; if (rsp.code == 0) { - if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->conn.applyIndex, &rsp) < 0) { + if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->info.conn.applyIndex, &rsp) < 0) { rsp.code = terrno; vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr()); } @@ -329,7 +329,7 @@ static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index}; + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .info.conn.applyIndex = cbMeta.index}; syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); vInfo("vgId:%d, alter vnode replica is confirmed, type:%s contLen:%d seq:%" PRIu64 " handle:%p", TD_VID(pVnode), @@ -359,7 +359,7 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index}; + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .info.conn.applyIndex = cbMeta.index}; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index b6e3fd2676..bd3781f9c7 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -310,7 +310,7 @@ static void uvHandleReq(SSvrConn* pConn) { } // set up conn info - SRpcConnInfo* pConnInfo = &(transMsg.info.connInfo); + SRpcConnInfo* pConnInfo = &(transMsg.info.conn); pConnInfo->clientIp = (uint32_t)(pConn->addr.sin_addr.s_addr); pConnInfo->clientPort = ntohs(pConn->addr.sin_port); tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user)); From 62f21b528eb844e5f06cd82e09ad098b720e1434 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 10:52:45 +0800 Subject: [PATCH 74/90] refactor: remove duplicate conninfo in SRpcMsg --- source/dnode/vnode/src/vnd/vnodeSync.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6a1f5c606d..f46114746e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -329,8 +329,9 @@ static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .info.conn.applyIndex = cbMeta.index}; + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + rpcMsg.info.conn.applyIndex = cbMeta.index; vInfo("vgId:%d, alter vnode replica is confirmed, type:%s contLen:%d seq:%" PRIu64 " handle:%p", TD_VID(pVnode), TMSG_INFO(pMsg->msgType), pMsg->contLen, cbMeta.seqNum, rpcMsg.info.handle); @@ -359,10 +360,11 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .info.conn.applyIndex = cbMeta.index}; + SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + rpcMsg.info.conn.applyIndex = cbMeta.index; tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg); } else { From f5db109e61c44f92e1d2e75edceac99057be579a Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Fri, 17 Jun 2022 11:37:10 +0800 Subject: [PATCH 75/90] fix(sync): get last max snapshot index --- include/libs/sync/sync.h | 1 + source/dnode/mnode/impl/src/mndSync.c | 8 +++-- source/dnode/mnode/impl/src/mndTrans.c | 8 +++-- source/libs/sync/inc/syncRaftCfg.h | 6 ++++ source/libs/sync/src/syncMain.c | 32 +++++++++++++++--- source/libs/sync/src/syncRaftCfg.c | 41 ++++++++++++++++++----- source/libs/sync/src/syncRaftLog.c | 26 +++++++------- source/libs/sync/src/syncSnapshot.c | 10 +++--- source/libs/sync/test/syncRaftCfgTest.cpp | 19 ++++++++++- 9 files changed, 114 insertions(+), 37 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index fc9d419b1f..f49030466e 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -192,6 +192,7 @@ bool syncEnvIsStart(); const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); +int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct SSnapshotMeta* sMeta); int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 3a243e36d3..388a9d82c4 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -69,7 +69,8 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) { SSnapshotMeta sMeta = {0}; - if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + // if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + if (syncGetSnapshotMetaByIndex(pMnode->syncMgmt.sync, cbMeta.index, &sMeta) == 0) { sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); } sdbWriteFile(pMnode->pSdb); @@ -89,7 +90,10 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; SSnapshotMeta sMeta = {0}; - if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + // if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + + SyncIndex snapshotIndex = sdbGetApplyIndex(pMnode->pSdb); + if (syncGetSnapshotMetaByIndex(pMnode->syncMgmt.sync, snapshotIndex, &sMeta) == 0) { sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 1631c9825b..a50d8da37e 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -22,8 +22,8 @@ #include "mndSync.h" #include "mndUser.h" -#define TRANS_VER_NUMBER 1 -#define TRANS_ARRAY_SIZE 8 +#define TRANS_VER_NUMBER 1 +#define TRANS_ARRAY_SIZE 8 #define TRANS_RESERVE_SIZE 64 static SSdbRaw *mndTransActionEncode(STrans *pTrans); @@ -1426,7 +1426,9 @@ void mndTransPullup(SMnode *pMnode) { } SSnapshotMeta sMeta = {0}; - if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + // if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) { + SyncIndex snapshotIndex = sdbGetApplyIndex(pMnode->pSdb); + if (syncGetSnapshotMetaByIndex(pMnode->syncMgmt.sync, snapshotIndex, &sMeta) == 0) { sdbSetCurConfig(pMnode->pSdb, sMeta.lastConfigIndex); } sdbWriteFile(pMnode->pSdb); diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index e72e1e7be7..cd64402738 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -29,6 +29,8 @@ extern "C" { #define CONFIG_FILE_LEN 1024 +#define MAX_CONFIG_INDEX_COUNT 512 + typedef struct SRaftCfg { SSyncCfg cfg; TdFilePtr pFile; @@ -36,6 +38,10 @@ typedef struct SRaftCfg { int8_t isStandBy; int8_t snapshotEnable; SyncIndex lastConfigIndex; + + SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; + int32_t configIndexCount; + } SRaftCfg; SRaftCfg *raftCfgOpen(const char *path); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 00786e5015..278288eb1e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -418,6 +418,29 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { return 0; } +int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct SSnapshotMeta* sMeta) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return -1; + } + assert(rid == pSyncNode->rid); + + ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1); + SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0]; + + for (int i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) { + if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex && + (pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotIndex) { + lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; + } + } + sMeta->lastConfigIndex = lastIndex; + sTrace("sync get snapshot meta by index:%ld lastConfigIndex:%ld", snapshotIndex, sMeta->lastConfigIndex); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return 0; +} + const char* syncGetMyRoleStr(int64_t rid) { const char* s = syncUtilState2String(syncGetMyRole(rid)); return s; @@ -2195,8 +2218,8 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE char* newStr = syncCfg2Str(&newSyncCfg); syncUtilJson2Line(oldStr); syncUtilJson2Line(newStr); - snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, %s --> %s", oldSyncCfg.replicaNum, - newSyncCfg.replicaNum, oldStr, newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%ld, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, pEntry->index, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); @@ -2212,8 +2235,8 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE char* newStr = syncCfg2Str(&newSyncCfg); syncUtilJson2Line(oldStr); syncUtilJson2Line(newStr); - snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d, %s --> %s", oldSyncCfg.replicaNum, - newSyncCfg.replicaNum, oldStr, newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d, index:%ld, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, pEntry->index, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); @@ -2295,6 +2318,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } // restore finish + // if only snapshot, a noop entry will be append, so syncLogLastIndex is always ok if (pEntry->index == ths->pLogStore->syncLogLastIndex(ths->pLogStore)) { if (ths->restoreFinish == false) { if (ths->pFsm->FpRestoreFinishCb != NULL) { diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 45e00aca2c..2d51f1f6f0 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -85,16 +85,11 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { } return pRoot; - /* - cJSON *pJson = cJSON_CreateObject(); - cJSON_AddItemToObject(pJson, "SSyncCfg", pRoot); - return pJson; - */ } char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -154,6 +149,16 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { snprintf(buf64, sizeof(buf64), "%ld", pRaftCfg->lastConfigIndex); cJSON_AddStringToObject(pRoot, "lastConfigIndex", buf64); + cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfg->configIndexCount); + cJSON *pIndexArr = cJSON_CreateArray(); + cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr); + for (int i = 0; i < pRaftCfg->configIndexCount; ++i) { + snprintf(buf64, sizeof(buf64), "%ld", (pRaftCfg->configIndexArr)[i]); + cJSON *pIndexObj = cJSON_CreateObject(); + cJSON_AddStringToObject(pIndexObj, "index", buf64); + cJSON_AddItemToArray(pIndexArr, pIndexObj); + } + cJSON *pJson = cJSON_CreateObject(); cJSON_AddItemToObject(pJson, "RaftCfg", pRoot); return pJson; @@ -161,7 +166,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -177,6 +182,9 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { raftCfg.isStandBy = meta.isStandBy; raftCfg.snapshotEnable = meta.snapshotEnable; raftCfg.lastConfigIndex = meta.lastConfigIndex; + raftCfg.configIndexCount = 1; + memset(raftCfg.configIndexArr, 0, sizeof(raftCfg.configIndexArr)); + raftCfg.configIndexArr[0] = -1; char *s = raftCfg2Str(&raftCfg); char buf[CONFIG_FILE_LEN] = {0}; @@ -207,7 +215,24 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { cJSON *pJsonLastConfigIndex = cJSON_GetObjectItem(pJson, "lastConfigIndex"); pRaftCfg->lastConfigIndex = atoll(cJSON_GetStringValue(pJsonLastConfigIndex)); - cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount"); + pRaftCfg->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount); + + cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr"); + int arraySize = cJSON_GetArraySize(pIndexArr); + assert(arraySize == pRaftCfg->configIndexCount); + + memset(pRaftCfg->configIndexArr, 0, sizeof(pRaftCfg->configIndexArr)); + for (int i = 0; i < arraySize; ++i) { + cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i); + assert(pIndexObj != NULL); + + cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index"); + assert(cJSON_IsString(pIndex)); + (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); + } + + cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 7c6d97a96e..87dfef5fdd 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -410,20 +410,18 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { } int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { - /* - SSyncLogStoreData* pData = pLogStore->data; - SWal* pWal = pData->pWal; - // assert(walCommit(pWal, index) == 0); - int32_t code = walCommit(pWal, index); - if (code != 0) { - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, - linuxErrMsg); ASSERT(0); - } - */ + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + // assert(walCommit(pWal, index) == 0); + int32_t code = walCommit(pWal, index); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t linuxErr = errno; + const char* linuxErrMsg = strerror(errno); + sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + ASSERT(0); + } return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ef2011cc26..4973d36295 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -421,7 +421,7 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -542,7 +542,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddStringToObject(pFromId, "addr", u64buf); { uint64_t u64 = pReceiver->fromId.addr; - cJSON *pTmp = pFromId; + cJSON * pTmp = pFromId; char host[128] = {0}; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); @@ -566,7 +566,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -671,8 +671,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *newStr = syncCfg2Str(&newSyncCfg); syncUtilJson2Line(oldStr); syncUtilJson2Line(newStr); - snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d, %s --> %s", oldSyncCfg.replicaNum, - newSyncCfg.replicaNum, oldStr, newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d, index:%ld, %s --> %s", + oldSyncCfg.replicaNum, newSyncCfg.replicaNum, pMsg->lastConfigIndex, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); diff --git a/source/libs/sync/test/syncRaftCfgTest.cpp b/source/libs/sync/test/syncRaftCfgTest.cpp index 8c6a704e2d..9460f0ad7a 100644 --- a/source/libs/sync/test/syncRaftCfgTest.cpp +++ b/source/libs/sync/test/syncRaftCfgTest.cpp @@ -27,6 +27,14 @@ SRaftCfg* createRaftCfg() { } pCfg->isStandBy = taosGetTimestampSec() % 100; + pCfg->configIndexCount = 5; + for (int i = 0; i < MAX_CONFIG_INDEX_COUNT; ++i) { + (pCfg->configIndexArr)[i] = -1; + } + for (int i = 0; i < pCfg->configIndexCount; ++i) { + (pCfg->configIndexArr)[i] = i * 100; + } + return pCfg; } @@ -100,6 +108,15 @@ void test5() { pCfg->isStandBy += 2; pCfg->snapshotEnable += 3; pCfg->lastConfigIndex += 1000; + + pCfg->configIndexCount = 5; + for (int i = 0; i < MAX_CONFIG_INDEX_COUNT; ++i) { + (pCfg->configIndexArr)[i] = -1; + } + for (int i = 0; i < pCfg->configIndexCount; ++i) { + (pCfg->configIndexArr)[i] = i * 100; + } + raftCfgPersist(pCfg); printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex); @@ -118,6 +135,6 @@ int main() { test3(); test4(); test5(); - + return 0; } From 28b32682246270336710717942cdda0f17d435b4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 11:40:18 +0800 Subject: [PATCH 76/90] fix(sync) add trace log --- source/libs/sync/src/syncMain.c | 8 ++++---- source/libs/sync/src/syncSnapshot.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 343a0e5597..dbf7bc94e0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2197,8 +2197,8 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE char* newStr = syncCfg2Str(&newSyncCfg); syncUtilJson2Line(oldStr); syncUtilJson2Line(newStr); - snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, %s --> %s", oldSyncCfg.replicaNum, - newSyncCfg.replicaNum, oldStr, newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%ld, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, pEntry->index, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); @@ -2214,8 +2214,8 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE char* newStr = syncCfg2Str(&newSyncCfg); syncUtilJson2Line(oldStr); syncUtilJson2Line(newStr); - snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d, %s --> %s", oldSyncCfg.replicaNum, - newSyncCfg.replicaNum, oldStr, newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d, index:%ld, %s --> %s", oldSyncCfg.replicaNum, + newSyncCfg.replicaNum, pEntry->index, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index ef2011cc26..c8d2866738 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -671,8 +671,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { char *newStr = syncCfg2Str(&newSyncCfg); syncUtilJson2Line(oldStr); syncUtilJson2Line(newStr); - snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d, %s --> %s", oldSyncCfg.replicaNum, - newSyncCfg.replicaNum, oldStr, newStr); + snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d, index:%ld, %s --> %s", + oldSyncCfg.replicaNum, newSyncCfg.replicaNum, pMsg->lastConfigIndex, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); From 18993d50e51c0f865a143a21cce001cdd85619dc Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 17 Jun 2022 12:10:51 +0800 Subject: [PATCH 77/90] refactor(tmq): offset storage --- source/client/src/tmq.c | 49 +++++++++++++++++++++++----------- source/dnode/vnode/src/tq/tq.c | 21 ++++++++++----- tests/test/c/tmqSim.c | 1 + 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 38b1872206..9b58ddc589 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -376,6 +376,9 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { } #endif + /*tscDebug("receive offset commit cb of %s on vg %d, offset is %ld", pParam->pOffset->subKey, pParam->->vgId, + * pOffset->version);*/ + // count down waiting rsp int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); ASSERT(waitingRspNum >= 0); @@ -421,10 +424,20 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + + tscDebug("consumer %ld begin commit for topic %s, vgNum %d", tmq->consumerId, pTopic->topicName, + (int32_t)taosArrayGetSize(pTopic->vgs)); + for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, i); - if (pVg->currentOffset < 0) { - /*if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {*/ + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + + tscDebug("consumer %ld begin commit for topic %s, vgId %d", tmq->consumerId, pTopic->topicName, pVg->vgId); + + /*if (pVg->currentOffset < 0) {*/ + if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { + tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); + continue; } STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); @@ -434,10 +447,12 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 } pOffset->type = TMQ_OFFSET__LOG; pOffset->version = pVg->currentOffset; - int32_t tlen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, tlen); - pOffset->subKey[tlen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + tlen + 1, pTopic->topicName); + + int32_t groupLen = strlen(tmq->groupId); + memcpy(pOffset->subKey, tmq->groupId, groupLen); + pOffset->subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); + int32_t len; int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, len, code); @@ -470,6 +485,9 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 .handle = NULL, }; + tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId, + pOffset->version); + // TODO: put into cb pVg->committedOffset = pVg->currentOffset; @@ -1039,14 +1057,13 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { if (pTopicCur->vgs) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); - if (vgNumCur == 0) break; for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); - tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %ld", tmq->consumerId, epoch, pVgCur->vgId, vgKey, + pVgCur->currentOffset); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } - break; } } @@ -1054,7 +1071,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientTopic topic = {0}; SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); topic.schema = pTopicEp->schema; - taosHashClear(pHash); topic.topicName = strdup(pTopicEp->topic); tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); @@ -1071,7 +1087,8 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { offset = *pOffset; } - tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch, + pVgEp->vgId, offset, vgKey); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, @@ -1350,10 +1367,10 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* /*strcpy(pReq->topic, pTopic->topicName);*/ /*strcpy(pReq->cgroup, tmq->groupId);*/ - int32_t tlen = strlen(tmq->groupId); - memcpy(pReq->subKey, tmq->groupId, tlen); - pReq->subKey[tlen] = TMQ_SEPARATOR; - strcpy(pReq->subKey + tlen + 1, pTopic->topicName); + int32_t groupLen = strlen(tmq->groupId); + memcpy(pReq->subKey, tmq->groupId, groupLen); + pReq->subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pReq->subKey + groupLen + 1, pTopic->topicName); pReq->withTbName = tmq->withTbName; pReq->timeout = timeout; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a7b3fc8a2f..c34738f083 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -124,17 +124,20 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { tDecoderClear(&decoder); if (offset.type == TMQ_OFFSET__SNAPSHOT) { - tqDebug("receive offset commit msg to %s, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, offset.uid, - offset.ts); + tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, + pTq->pVnode->config.vgId, offset.uid, offset.ts); } else if (offset.type == TMQ_OFFSET__LOG) { - tqDebug("receive offset commit msg to %s, offset(type:log) version: %ld", offset.subKey, offset.version); + tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey, + pTq->pVnode->config.vgId, offset.version); } else { ASSERT(0); } - - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - ASSERT(0); - return -1; + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); + if (pOffset == NULL || pOffset->version < offset.version) { + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + ASSERT(0); + return -1; + } } return 0; @@ -155,6 +158,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); if (pOffset != NULL) { ASSERT(pOffset->type == TMQ_OFFSET__LOG); + tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey, + pTq->pVnode->config.vgId, pOffset->version); fetchOffset = pOffset->version + 1; } else { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { @@ -166,6 +171,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { pTq->pVnode->config.vgId, pReq->subKey); return -1; } + tqDebug("consumer %ld, restore offset of %s on vg %d failed, config is %ld, set to %ld", consumerId, pReq->subKey, + pTq->pVnode->config.vgId, pReq->currentOffset, fetchOffset); } } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 8c2a660dd6..3737484a2c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -513,6 +513,7 @@ void* consumeThreadFunc(void* param) { /*tmq_commit(pInfo->tmq, NULL, 0);*/ tmq_commit_sync(pInfo->tmq, NULL); taosFprintfFile(g_fp, "tmq_commit() manual commit over.\n"); + pPrint("tmq_commit() manual commit over.\n"); } err = tmq_unsubscribe(pInfo->tmq); From 2df98609e8bd4d9213554967c972ccfa8356da41 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 17 Jun 2022 13:55:25 +0800 Subject: [PATCH 78/90] feat: the unique function and tail function are rewritten as the corresponding clauses --- include/libs/nodes/nodes.h | 1 + include/libs/nodes/querynodes.h | 11 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/parser/src/parAstCreater.c | 2 + source/libs/parser/src/parTranslater.c | 125 +++++++++++++++---- source/libs/parser/test/parSelectTest.cpp | 2 + source/libs/planner/test/planBasicTest.cpp | 2 + source/libs/planner/test/planOrderByTest.cpp | 8 ++ source/libs/planner/test/planSetOpTest.cpp | 12 +- tests/system-test/2-query/tail.py | 14 +-- tests/system-test/2-query/unique.py | 28 ++--- 11 files changed, 157 insertions(+), 50 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3620f45408..6f239cc892 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -102,6 +102,7 @@ typedef enum ENodeType { QUERY_NODE_EXPLAIN_OPTIONS, QUERY_NODE_STREAM_OPTIONS, QUERY_NODE_LEFT_VALUE, + QUERY_NODE_COLUMN_REF, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 944b1f4b2e..3a0d3178a3 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -65,11 +65,16 @@ typedef struct SColumnNode { char tableName[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; - SNode* pProjectRef; - int16_t dataBlockId; - int16_t slotId; + // SNode* pProjectRef; + int16_t dataBlockId; + int16_t slotId; } SColumnNode; +typedef struct SColumnRefNode { + ENodeType type; + char colName[TSDB_COL_NAME_LEN]; +} SColumnRefNode; + typedef struct STargetNode { ENodeType type; int16_t dataBlockId; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index e2f4dd26f6..6d04e7a959 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -88,6 +88,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStreamOptions)); case QUERY_NODE_LEFT_VALUE: return makeNode(type, sizeof(SLeftValueNode)); + case QUERY_NODE_COLUMN_REF: + return makeNode(type, sizeof(SColumnDefNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 8a7987e4ec..03b5f4332d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -648,6 +648,8 @@ SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrder CHECK_PARSER_STATUS(pCxt); if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { ((SSelectStmt*)pStmt)->pOrderByList = pOrderByList; + } else { + ((SSetOperator*)pStmt)->pOrderByList = pOrderByList; } return pStmt; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 814c247e09..57c464f09c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -36,6 +36,7 @@ typedef struct STranslateContext { int32_t currLevel; ESqlClause currClause; SSelectStmt* pCurrSelectStmt; + SSetOperator* pCurrSetOperator; SCmdMsgInfo* pCmdMsg; SHashObj* pDbs; SHashObj* pTables; @@ -432,7 +433,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SColumnNode** pColRef) { SColumnNode* pCol = *pColRef; - pCol->pProjectRef = (SNode*)pExpr; + // pCol->pProjectRef = (SNode*)pExpr; if (NULL == pExpr->pAssociation) { pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); } @@ -613,17 +614,36 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod return DEAL_RES_CONTINUE; } -static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol) { - SNodeList* pProjectionList = pCxt->pCurrSelectStmt->pProjectionList; +static SNodeList* getProjectListFromCxt(STranslateContext* pCxt) { + if (NULL != pCxt->pCurrSelectStmt) { + return pCxt->pCurrSelectStmt->pProjectionList; + } else if (NULL != pCxt->pCurrSetOperator) { + return pCxt->pCurrSetOperator->pProjectionList; + } else { + return NULL; + } +} + +static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol, bool* pFound) { + SNodeList* pProjectionList = getProjectListFromCxt(pCxt); SNode* pNode; FOREACH(pNode, pProjectionList) { SExprNode* pExpr = (SExprNode*)pNode; if (0 == strcmp((*pCol)->colName, pExpr->aliasName)) { - setColumnInfoByExpr(NULL, pExpr, pCol); - return true; + SColumnRefNode* pColRef = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF); + if (NULL == pColRef) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + strcpy(pColRef->colName, pExpr->aliasName); + nodesDestroyNode(*(SNode**)pCol); + *(SNode**)pCol = (SNode*)pColRef; + *pFound = true; + return DEAL_RES_CONTINUE; } } - return false; + *pFound = false; + return DEAL_RES_CONTINUE; } static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { @@ -638,9 +658,11 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { } else { bool found = false; if (SQL_CLAUSE_ORDER_BY == pCxt->currClause) { - found = translateColumnUseAlias(pCxt, pCol); + res = translateColumnUseAlias(pCxt, pCol, &found); + } + if (DEAL_RES_ERROR != res && !found) { + res = translateColumnWithoutPrefix(pCxt, pCol); } - res = (found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol)); } return res; } @@ -1803,11 +1825,11 @@ static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pPro } else if (0 == pos || pos > LIST_LENGTH(pProjectionList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT); } else { - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF); if (NULL == pCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); } - setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos - 1), &pCol); + strcpy(pCol->colName, ((SExprNode*)nodesListGetNode(pProjectionList, pos - 1))->aliasName); ((SOrderByExprNode*)pNode)->pExpr = (SNode*)pCol; nodesDestroyNode(pExpr); } @@ -1822,16 +1844,15 @@ static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pPro static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; int32_t code = translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - if (!other) { - return TSDB_CODE_SUCCESS; - } - pCxt->currClause = SQL_CLAUSE_ORDER_BY; - code = translateExprList(pCxt, pSelect->pOrderByList); if (TSDB_CODE_SUCCESS == code) { - code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); + if (!other) { + return TSDB_CODE_SUCCESS; + } + pCxt->currClause = SQL_CLAUSE_ORDER_BY; + code = translateExprList(pCxt, pSelect->pOrderByList); + if (TSDB_CODE_SUCCESS == code) { + code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); + } } return code; } @@ -2290,6 +2311,39 @@ static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +typedef struct SReplaceOrderByAliasCxt { + STranslateContext* pTranslateCxt; + SNodeList* pProjectionList; +} SReplaceOrderByAliasCxt; + +static EDealRes replaceOrderByAliasImpl(SNode** pNode, void* pContext) { + SReplaceOrderByAliasCxt* pCxt = pContext; + if (QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { + SNodeList* pProjectionList = pCxt->pProjectionList; + SNode* pProject = NULL; + FOREACH(pProject, pProjectionList) { + SExprNode* pExpr = (SExprNode*)pProject; + if (0 == strcmp(((SColumnRefNode*)*pNode)->colName, pExpr->aliasName)) { + SNode* pNew = nodesCloneNode(pProject); + if (NULL == pNew) { + pCxt->pTranslateCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + nodesDestroyNode(*pNode); + *pNode = pNew; + return DEAL_RES_CONTINUE; + } + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t replaceOrderByAlias(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList) { + SReplaceOrderByAliasCxt cxt = {.pTranslateCxt = pCxt, .pProjectionList = pProjectionList}; + nodesRewriteExprsPostOrder(pOrderByList, replaceOrderByAliasImpl, &cxt); + return pCxt->errCode; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrSelectStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect->pFromTable); @@ -2324,12 +2378,15 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = rewriteUniqueStmt(pCxt, pSelect); } - if (TSDB_CODE_SUCCESS == code) { - code = rewriteTailStmt(pCxt, pSelect); - } + // if (TSDB_CODE_SUCCESS == code) { + // code = rewriteTailStmt(pCxt, pSelect); + // } if (TSDB_CODE_SUCCESS == code) { code = rewriteTimelineFunc(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = replaceOrderByAlias(pCxt, pSelect->pProjectionList, pSelect->pOrderByList); + } return code; } @@ -2366,7 +2423,7 @@ static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType d return TSDB_CODE_SUCCESS; } -static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* pSetOperator) { +static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pSetOperator) { SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); if (LIST_LENGTH(pLeftProjections) != LIST_LENGTH(pRightProjections)) { @@ -2401,6 +2458,23 @@ static uint8_t calcSetOperatorPrecision(SSetOperator* pSetOperator) { return calcPrecision(getStmtPrecision(pSetOperator->pLeft), getStmtPrecision(pSetOperator->pRight)); } +static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pSetOperator) { + bool other; + int32_t code = translateOrderByPosition(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList, &other); + if (TSDB_CODE_SUCCESS == code) { + if (other) { + pCxt->currClause = SQL_CLAUSE_ORDER_BY; + pCxt->pCurrSelectStmt = NULL; + pCxt->pCurrSetOperator = pSetOperator; + code = translateExprList(pCxt, pSetOperator->pOrderByList); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = replaceOrderByAlias(pCxt, pSetOperator->pProjectionList, pSetOperator->pOrderByList); + } + return code; +} + static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetOperator) { int32_t code = translateQuery(pCxt, pSetOperator->pLeft); if (TSDB_CODE_SUCCESS == code) { @@ -2411,7 +2485,10 @@ static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetO } if (TSDB_CODE_SUCCESS == code) { pSetOperator->precision = calcSetOperatorPrecision(pSetOperator); - code = translateSetOperatorImpl(pCxt, pSetOperator); + code = translateSetOperProject(pCxt, pSetOperator); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateSetOperOrderBy(pCxt, pSetOperator); } return code; } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 97978a90f4..a69b012574 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -362,6 +362,8 @@ TEST_F(ParserSelectTest, setOperator) { run("(SELECT * FROM t1) UNION ALL (SELECT * FROM t1)"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); + + run("SELECT c1, c2 FROM t1 UNION ALL SELECT c1 as a, c2 as b FROM t1 ORDER BY c1"); } TEST_F(ParserSelectTest, informationSchema) { diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 0304c6708d..c075bfeee8 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -62,6 +62,8 @@ TEST_F(PlanBasicTest, uniqueFunc) { run("SELECT UNIQUE(c2 + 10) FROM t1 WHERE c1 > 10"); run("SELECT UNIQUE(c2 + 10), ts, c2 FROM t1 WHERE c1 > 10"); + + run("SELECT UNIQUE(c1) a FROM t1 ORDER BY a"); } TEST_F(PlanBasicTest, tailFunc) { diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp index 39e93fcff9..b065ad0a59 100644 --- a/source/libs/planner/test/planOrderByTest.cpp +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -27,6 +27,8 @@ TEST_F(PlanOrderByTest, basic) { run("SELECT c1 FROM t1 ORDER BY c1"); // ORDER BY key is not in the projection list run("SELECT c1 FROM t1 ORDER BY c2"); + + run("SELECT c1 + 10 AS a FROM t1 ORDER BY a"); } TEST_F(PlanOrderByTest, expr) { @@ -41,6 +43,12 @@ TEST_F(PlanOrderByTest, nullsOrder) { run("SELECT * FROM t1 ORDER BY c1 DESC NULLS FIRST"); } +TEST_F(PlanOrderByTest, withGroupBy) { + useDb("root", "test"); + + run("SELECT SUM(c1) AS a FROM t1 GROUP BY c2 ORDER BY a"); +} + TEST_F(PlanOrderByTest, stable) { useDb("root", "test"); diff --git a/source/libs/planner/test/planSetOpTest.cpp b/source/libs/planner/test/planSetOpTest.cpp index 5568d3cfbb..bf26f55c02 100644 --- a/source/libs/planner/test/planSetOpTest.cpp +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -23,9 +23,9 @@ class PlanSetOpTest : public PlannerTestBase {}; TEST_F(PlanSetOpTest, unionAll) { useDb("root", "test"); - // sql 1: single UNION ALL operator + // single UNION ALL operator run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20"); - // sql 2: multi UNION ALL operator + // multi UNION ALL operator run("SELECT c1, c2 FROM t1 WHERE c1 > 10 " "UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 " "UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 30"); @@ -46,6 +46,14 @@ TEST_F(PlanSetOpTest, unionAllWithSubquery) { run("SELECT ts FROM (SELECT ts FROM st1) UNION ALL SELECT ts FROM (SELECT ts FROM st1)"); } +TEST_F(PlanSetOpTest, unionAllWithOrderBy) { + useDb("root", "test"); + + run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 ORDER BY c1"); + + run("SELECT c1, c2 FROM t1 WHERE c1 > 10 UNION ALL SELECT c1, c2 FROM t1 WHERE c1 > 20 ORDER BY 1"); +} + TEST_F(PlanSetOpTest, union) { useDb("root", "test"); diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index bb11fbbef5..0e2110f912 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -188,8 +188,8 @@ class TDTestCase: def check_tail_table(self , tbname , col_name , tail_rows , offset): tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}" - #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" - equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" + equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts" + #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}" tdSql.query(tail_sql) tail_result = tdSql.queryResult @@ -294,19 +294,19 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, None) - tdSql.query("select tail(c1,3,2) from ct4 where c1 >2 ") + tdSql.query("select tail(c1,3,2) from ct4 where c1 >2 order by 1") tdSql.checkData(0, 0, 5) tdSql.checkData(1, 0, 6) tdSql.checkData(2, 0, 7) - tdSql.query("select tail(c1,2,1) from ct4 where c2 between 0 and 99999") + tdSql.query("select tail(c1,2,1) from ct4 where c2 between 0 and 99999 order by 1") tdSql.checkData(0, 0, 1) tdSql.checkData(1, 0, 2) # tail with union all tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct1") tdSql.checkRows(15) - tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct2") + tdSql.query("select tail(c1,2,1) from ct4 union all select c1 from ct2 order by 1") tdSql.checkRows(2) tdSql.checkData(0, 0, 0) tdSql.checkData(1, 0, 1) @@ -335,7 +335,7 @@ class TDTestCase: tdSql.execute(f" insert into ttb1 values({ts_value} , {i})") tdSql.execute(f" insert into ttb2 values({ts_value} , {i})") - tdSql.query("select tail(tb2.num,3,2) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.query("select tail(tb2.num,3,2) from tb1, tb2 where tb1.ts=tb2.ts order by 1 desc") tdSql.checkRows(3) tdSql.checkData(0,0,7) tdSql.checkData(1,0,6) @@ -343,7 +343,7 @@ class TDTestCase: # nest query # tdSql.query("select tail(c1,2) from (select c1 from ct1)") - tdSql.query("select c1 from (select tail(c1,2) c1 from ct4)") + tdSql.query("select c1 from (select tail(c1,2) c1 from ct4) order by 1 nulls first") tdSql.checkRows(2) tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, 0) diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index f48202010a..f910ff1439 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -289,13 +289,13 @@ class TDTestCase: tdSql.query("select unique(c1) from ct4 where c1 is null") tdSql.checkData(0, 0, None) - tdSql.query("select unique(c1) from ct4 where c1 >2") - tdSql.checkData(0, 0, 8) - tdSql.checkData(1, 0, 7) - tdSql.checkData(2, 0, 6) - tdSql.checkData(5, 0, 3) + tdSql.query("select unique(c1) from ct4 where c1 >2 order by 1") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 4) + tdSql.checkData(2, 0, 5) + tdSql.checkData(5, 0, 8) - tdSql.query("select unique(c1) from ct4 where c2 between 0 and 99999") + tdSql.query("select unique(c1) from ct4 where c2 between 0 and 99999 order by 1 desc") tdSql.checkData(0, 0, 8) tdSql.checkData(1, 0, 7) tdSql.checkData(2, 0, 6) @@ -336,23 +336,23 @@ class TDTestCase: tdSql.execute(f" insert into ttb1 values({ts_value} , {i})") tdSql.execute(f" insert into ttb2 values({ts_value} , {i})") - tdSql.query("select unique(tb2.num) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.query("select unique(tb2.num) from tb1, tb2 where tb1.ts=tb2.ts order by 1") tdSql.checkRows(10) tdSql.checkData(0,0,0) tdSql.checkData(1,0,1) tdSql.checkData(2,0,2) tdSql.checkData(9,0,9) - tdSql.query("select unique(tb2.num) from tb1, tb2 where tb1.ts=tb2.ts union all select unique(tb1.num) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.query("select unique(tb2.num) from tb1, tb2 where tb1.ts=tb2.ts union all select unique(tb1.num) from tb1, tb2 where tb1.ts=tb2.ts order by 1") tdSql.checkRows(20) tdSql.checkData(0,0,0) - tdSql.checkData(1,0,1) - tdSql.checkData(2,0,2) - tdSql.checkData(9,0,9) + tdSql.checkData(2,0,1) + tdSql.checkData(4,0,2) + tdSql.checkData(18,0,9) # nest query # tdSql.query("select unique(c1) from (select c1 from ct1)") - tdSql.query("select c1 from (select unique(c1) c1 from ct4)") + tdSql.query("select c1 from (select unique(c1) c1 from ct4) order by 1 desc nulls first") tdSql.checkRows(10) tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, 8) @@ -367,7 +367,7 @@ class TDTestCase: tdSql.checkData(0, 0, 45) tdSql.checkData(1, 0, 45) - tdSql.query("select 1-abs(c1) from (select unique(c1) c1 from ct4)") + tdSql.query("select 1-abs(c1) from (select unique(c1) c1 from ct4) order by 1 nulls first") tdSql.checkRows(10) tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, -7.000000000) @@ -422,7 +422,7 @@ class TDTestCase: f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.query("select unique(c2) from sub1_bound") + tdSql.query("select unique(c2) from sub1_bound order by 1 desc") tdSql.checkRows(5) tdSql.checkData(0,0,9223372036854775807) From 43f8f34b0f7088167854b0207869055d003fa2a4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Jun 2022 14:02:51 +0800 Subject: [PATCH 79/90] refactor: adjust logs --- source/dnode/mnode/impl/src/mndMain.c | 8 ++-- source/dnode/mnode/impl/src/mndTrans.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 9 ++-- source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/vnd/vnodeSync.c | 3 +- source/libs/sync/src/syncAppendEntries.c | 4 +- source/libs/sync/src/syncAppendEntriesReply.c | 4 +- source/libs/sync/src/syncCommit.c | 2 +- source/libs/sync/src/syncMain.c | 43 ++++++++++--------- source/libs/sync/src/syncRaftLog.c | 4 +- source/libs/sync/src/syncRespMgr.c | 6 +-- source/libs/sync/src/syncSnapshot.c | 38 ++++++++-------- 12 files changed, 64 insertions(+), 61 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 7b5025a986..bc418e0f53 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -510,8 +510,7 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (IsReq(pMsg) && pMsg->msgType != TDMT_MND_MQ_TIMER && pMsg->msgType != TDMT_MND_TELEM_TIMER && pMsg->msgType != TDMT_MND_TRANS_TIMER) { - mError("msg:%p, failed to check mnode state since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, - TMSG_INFO(pMsg->msgType)); + mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); SEpSet epSet = {0}; mndGetMnodeEpSet(pMsg->info.node, &epSet); @@ -534,7 +533,8 @@ static int32_t mndCheckMsgContent(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0; - mError("msg:%p, failed to check msg content, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); + mError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen, + pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; return -1; } @@ -558,7 +558,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mTrace("msg:%p, won't response immediately since in progress", pMsg); } else if (code == 0) { - mTrace("msg:%p, successfully processed and response", pMsg); + mTrace("msg:%p, successfully processed", pMsg); } else { mError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 858c6f956f..11e07cb9fa 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -661,7 +661,7 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mDebug("trans:%d, sync to other mnodes", pTrans->id); + mDebug("trans:%d, sync to other mnodes, stage:%s", pTrans->id, mndTransStr(pTrans->stage)); int32_t code = mndSyncPropose(pMnode, pRaw, pTrans->id); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index cec83d1af5..94ddbcd409 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1058,7 +1058,7 @@ int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t del static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t newDnodeId) { - mDebug("vgId:%d, will add 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); + mDebug("vgId:%d, will add 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId); SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica]; pVgroup->replica++; @@ -1074,7 +1074,7 @@ static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDb static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t delDnodeId) { - mDebug("vgId:%d, will remove 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); + mDebug("vgId:%d, will remove 1 vnode, replica:%d dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId); SVnodeGid *pGid = NULL; SVnodeGid delGid = {0}; @@ -1116,7 +1116,8 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, memcpy(&newVg, pVgroup, sizeof(SVgObj)); mInfo("vgId:%d, vgroup info before redistribute, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { - mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); + mInfo("vgId:%d, vnode:%d dnode:%d role:%s", newVg.vgId, i, newVg.vnodeGid[i].dnodeId, + syncStr(newVg.vnodeGid[i].role)); } if (pNew1 != NULL && pOld1 != NULL) { @@ -1198,7 +1199,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { goto _OVER; } - mInfo("vgId:%d, start to redistribute to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); + mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) goto _OVER; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a6339125c4..8cfebe1b98 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -628,7 +628,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { void * tagData = NULL; if (param->val == NULL) { - metaError("vgId:%d failed to filter NULL data", TD_VID(pMeta->pVnode)); + metaError("vgId:%d, failed to filter NULL data", TD_VID(pMeta->pVnode)); return -1; } else { if (IS_VAR_DATA_TYPE(param->type)) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f46114746e..fded723f1a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -61,6 +61,7 @@ static int32_t vnodeSetStandBy(SVnode *pVnode) { return -1; } + vInfo("vgId:%d, start to transfer leader", TD_VID(pVnode)); if (syncLeaderTransfer(pVnode->sync) != 0) { vError("vgId:%d, failed to transfer leader since:%s", TD_VID(pVnode), terrstr()); return -1; @@ -297,7 +298,7 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ret = -1; } - if (ret != 0) { + if (ret != 0 && terrno == 0) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } return ret; diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 7b2f79e24c..a8cb3d0e69 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -713,7 +713,7 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { // delete confict entries code = ths->pLogStore->syncLogTruncate(ths->pLogStore, delBegin); ASSERT(code == 0); - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu log truncate, from %ld to %ld", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, delBegin, delEnd); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); @@ -1062,7 +1062,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ths->commitIndex = snapshot.lastApplyIndex; sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu commit by snapshot from index:%ld to index:%ld", + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu commit by snapshot from index:%ld to index:%ld", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, commitBegin, commitEnd); } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f934f9a268..eabd14bf28 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -190,7 +190,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (gRaftDetailLog) { char* s = snapshotSender2Str(pSender); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d start sender first time, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d start sender first time, " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu " @@ -201,7 +201,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries taosMemoryFree(s); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d start sender first time, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d start sender first time, " "lastApplyIndex:%ld " "lastApplyTerm:%lu lastConfigIndex:%ld privateTerm:%lu", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, host, port, diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index d010728c78..d563e5aacc 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -56,7 +56,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex commitEnd = snapshot.lastApplyIndex; pSyncNode->commitIndex = snapshot.lastApplyIndex; - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu commit by snapshot from index:%ld to index:%ld", + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu commit by snapshot from index:%ld to index:%ld", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, snapshot.lastApplyIndex); } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 3e5dcfea42..2b06673a62 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -414,7 +414,7 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { assert(rid == pSyncNode->rid); sMeta->lastConfigIndex = pSyncNode->pRaftCfg->lastConfigIndex; - sTrace("sync get snapshot meta: lastConfigIndex:%ld", pSyncNode->pRaftCfg->lastConfigIndex); + sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->pRaftCfg->lastConfigIndex); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return 0; @@ -437,7 +437,8 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct } } sMeta->lastConfigIndex = lastIndex; - sTrace("sync get snapshot meta by index:%ld lastConfigIndex:%ld", snapshotIndex, sMeta->lastConfigIndex); + sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lastConfigIndex:%" PRId64, pSyncNode->vgId, snapshotIndex, + sMeta->lastConfigIndex); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return 0; @@ -598,7 +599,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { return -1; } assert(rid == pSyncNode->rid); - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), pMsg->msgType); ret = syncNodePropose(pSyncNode, pMsg, isWeak); @@ -609,7 +610,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu propose msgType:%s,%d", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pMsg->msgType), pMsg->msgType); @@ -857,7 +858,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // snapshot meta // pSyncNode->sMeta.lastConfigIndex = -1; - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu sync open", pSyncNode->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu sync open", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm); return pSyncNode; @@ -905,7 +906,7 @@ void syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodeClose(SSyncNode* pSyncNode) { - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu sync close", pSyncNode->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu sync close", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm); int32_t ret; @@ -1294,7 +1295,7 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { int len = 256; char* s = (char*)taosMemoryMalloc(len); snprintf(s, len, - "syncNode: vgId:%d currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " + "syncNode: vgId:%d, currentTerm:%lu, commitIndex:%ld, state:%d %s, isStandBy:%d, " "electTimerLogicClock:%lu, " "electTimerLogicClockUser:%lu, " "electTimerMS:%d, replicaNum:%d", @@ -1345,7 +1346,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { oldSenders[i] = (pSyncNode->senders)[i]; - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu save senders %d, %p, privateTerm:%lu", + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu save senders %d, %p, privateTerm:%lu", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, i, oldSenders[i], oldSenders[i]->privateTerm); if (gRaftDetailLog) { @@ -1400,7 +1401,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu reset sender for %lu, newIndex:%d, %s:%d, %p, " "privateTerm:%lu", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j], @@ -1413,7 +1414,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; (pSyncNode->senders)[i]->replicaIndex = i; sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu udpate replicaIndex from %d to %d, %s:%d, %p, " "reset:%d", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); @@ -1426,7 +1427,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu create new sender %p replicaIndex:%d, privateTerm:%lu", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, (pSyncNode->senders)[i], i, (pSyncNode->senders)[i]->privateTerm); } @@ -1436,7 +1437,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { snapshotSenderDestroy(oldSenders[i]); - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu delete old sender %p replicaIndex:%d", + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu delete old sender %p replicaIndex:%d", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, oldSenders[i], i); oldSenders[i] = NULL; @@ -1509,7 +1510,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu become follower, isStandBy:%d, replicaNum:%d, " "restoreFinish:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, @@ -1551,7 +1552,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { pSyncNode->restoreFinish = false; sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu become leader, isStandBy:%d, replicaNum:%d, " "restoreFinish:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, @@ -1857,7 +1858,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { if (pSyncNode->FpEqMsg != NULL) { int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { - sError("vgId:%d sync enqueue ping msg error, code:%d", pSyncNode->vgId, code); + sError("vgId:%d, sync enqueue ping msg error, code:%d", pSyncNode->vgId, code); rpcFreeCont(rpcMsg.pCont); syncTimeoutDestroy(pSyncMsg); return; @@ -1891,7 +1892,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { if (pSyncNode->FpEqMsg != NULL) { int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { - sError("vgId:%d sync enqueue elect msg error, code:%d", pSyncNode->vgId, code); + sError("vgId:%d, sync enqueue elect msg error, code:%d", pSyncNode->vgId, code); rpcFreeCont(rpcMsg.pCont); syncTimeoutDestroy(pSyncMsg); return; @@ -1929,7 +1930,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (pSyncNode->FpEqMsg != NULL) { int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { - sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + sError("vgId:%d, sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); rpcFreeCont(rpcMsg.pCont); syncTimeoutDestroy(pSyncMsg); return; @@ -2129,12 +2130,12 @@ const char* syncStr(ESyncState state) { static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu begin leader transfer", ths->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu begin leader transfer", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm); if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu maybe leader transfer to %s:%d %lu", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); @@ -2275,7 +2276,7 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { int32_t code = 0; ESyncState state = flag; - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, beginIndex, endIndex, syncUtilState2String(state)); @@ -2327,7 +2328,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->pFsm->FpRestoreFinishCb(ths->pFsm); } ths->restoreFinish = true; - sDebug("vgId:%d sync event %s commitIndex:%ld currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, + sDebug("vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu restore finish, %s, index:%ld", ths->vgId, syncUtilState2String(ths->state), ths->commitIndex, ths->pRaftStore->currentTerm, syncUtilState2String(ths->state), pEntry->index); } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 87dfef5fdd..8fff7ae6de 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -164,7 +164,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr walFsync(pWal, true); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu write index:%ld, isStandBy:%d, msgType:%s,%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu write index:%ld, isStandBy:%d, msgType:%s,%d, " "originalRpcType:%s,%d", pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->commitIndex, pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, @@ -325,7 +325,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { walFsync(pWal, true); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu old write index:%ld, isStandBy:%d, msgType:%s,%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu old write index:%ld, isStandBy:%d, msgType:%s,%d, " "originalRpcType:%s,%d", pData->pSyncNode->vgId, syncUtilState2String(pData->pSyncNode->state), pData->pSyncNode->commitIndex, pData->pSyncNode->pRaftStore->currentTerm, pEntry->index, pData->pSyncNode->pRaftCfg->isStandBy, diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 551c2611b0..831e286d76 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -47,7 +47,7 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { SSyncNode *pSyncNode = pObj->data; sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu resp mgr add, msgType:%s,%d seq:%lu handle:%p ahandle:%p", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); @@ -74,7 +74,7 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { SSyncNode *pSyncNode = pObj->data; sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu resp mgr get, msgType:%s,%d seq:%lu handle:%p " "ahandle:%p", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, @@ -96,7 +96,7 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu SSyncNode *pSyncNode = pObj->data; sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu resp mgr get and del, msgType:%s,%d seq:%lu handle:%p " "ahandle:%p", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, pSyncNode->pRaftStore->currentTerm, TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 4973d36295..7c8abfe494 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -141,7 +141,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " @@ -153,7 +153,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d begin seq:%d ack:%d " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", @@ -287,7 +287,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu send " @@ -299,7 +299,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d finish seq:%d ack:%d " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", @@ -310,7 +310,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { } } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d sending seq:%d ack:%d " "lastApplyIndex:%ld " "lastApplyTerm:%lu " "lastConfigIndex:%ld privateTerm:%lu", @@ -349,7 +349,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d " "privateTerm:%lu send " "msg:%s", pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, @@ -358,7 +358,7 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot send to %s:%d resend seq:%d ack:%d " "privateTerm:%lu", pSender->pSyncNode->vgId, syncUtilState2String(pSender->pSyncNode->state), pSender->pSyncNode->commitIndex, pSender->pSyncNode->pRaftStore->currentTerm, host, port, pSender->seq, pSender->ack, pSender->privateTerm); @@ -594,7 +594,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d begin ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d begin ack:%d, " "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", @@ -604,7 +604,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d begin ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d begin ack:%d, " "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld privateTerm:%lu", @@ -648,7 +648,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { bool isDrop; if (IamInNew) { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu update config by snapshot, lastIndex:%ld, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu update config by snapshot, lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld ", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, @@ -656,7 +656,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { syncNodeUpdateConfig(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex, &isDrop); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu do not update config by snapshot, I am not in " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu do not update config by snapshot, I am not in " "newCfg, " "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld ", @@ -694,7 +694,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *logSimpleStr = logStoreSimple2Str(pSyncNode->pLogStore); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d finish, update log begin " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d finish, update log begin " "index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu, raft log:%s", @@ -704,7 +704,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { taosMemoryFree(logSimpleStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d finish, update log begin " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d finish, update log begin " "index:%ld, " "snapshot.lastApplyIndex:%ld, " "snapshot.lastApplyTerm:%lu, snapshot.lastConfigIndex:%ld, privateTerm:%lu", @@ -721,7 +721,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d end ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d end ack:%d, " "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, @@ -730,7 +730,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d end ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d end ack:%d, " "lastIndex:%ld, lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", pReceiver->pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->commitIndex, @@ -750,7 +750,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d force close ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d force close ack:%d, " "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv " @@ -761,7 +761,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d force close ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d force close ack:%d, " "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", @@ -787,7 +787,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { if (gRaftDetailLog) { char *msgStr = syncSnapshotSend2Str(pMsg); sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, " "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu, recv msg:%s", @@ -797,7 +797,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { taosMemoryFree(msgStr); } else { sDebug( - "vgId:%d sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, " + "vgId:%d, sync event %s commitIndex:%ld currentTerm:%lu snapshot recv from %s:%d receiving ack:%d, " "lastIndex:%ld, " "lastTerm:%lu, " "lastConfigIndex:%ld, privateTerm:%lu", From 9c6a9f1c9299ad94e5cda155dbd2e268bdb4be5c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Jun 2022 14:25:53 +0800 Subject: [PATCH 80/90] fix(query): set correct source status in merge sort. --- source/libs/executor/src/sortoperator.c | 6 ++---- source/libs/executor/src/tsort.c | 14 ++++++++++---- source/os/src/osSocket.c | 3 +-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 81899b68cd..77eeb24210 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -392,10 +392,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, pInfo->bufPageSize = getProperSortPageSize(rowSize); - uint32_t numOfSources = taosArrayGetSize(pSortInfo); - numOfSources = TMAX(4, numOfSources); - - pInfo->sortBufSize = numOfSources * pInfo->bufPageSize; + // one additional is reserved for merged result. + pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); pOperator->fpSet = createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 1502387360..00e07c7199 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -195,6 +195,11 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); } +static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) { + pSource->src.rowIndex = -1; + ++pHandle->numOfCompletedSources; +} + static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { cmpParam->pSources = taosArrayGet(pSources, startIndex); cmpParam->numOfSources = (endIndex - startIndex + 1); @@ -205,8 +210,10 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { SSortSource* pSource = cmpParam->pSources[i]; + // set current source is done if (taosArrayGetSize(pSource->pageIdList) == 0) { - return TSDB_CODE_SUCCESS; + setCurrentSourceIsDone(pSource, pHandle); + continue; } SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); @@ -233,10 +240,9 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int SSortSource* pSource = cmpParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); - // set current source id done + // set current source is done if (pSource->src.pBlock == NULL) { - pSource->src.rowIndex = -1; - ++pHandle->numOfCompletedSources; + setCurrentSourceIsDone(pSource, pHandle); } } } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 4a0d9e2866..4d61e7036d 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -946,8 +946,7 @@ int32_t taosGetFqdn(char *fqdn) { #endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - // printf("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); - assert(0); + fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } From 52bbbe59d2871b44fedd9b6b23770f8b74dd57dc Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 17 Jun 2022 14:39:34 +0800 Subject: [PATCH 81/90] fix: a problem of super table order by --- source/libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/planner/src/planSpliter.c | 6 ++++-- source/libs/planner/test/planOrderByTest.cpp | 4 ++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 68f9e9d36d..9758da7899 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -285,7 +285,7 @@ class MockCatalogServiceImpl { } void createSmaIndex(const SMCreateSmaReq* pReq) { - STableIndexInfo info; + STableIndexInfo info = {0}; info.intervalUnit = pReq->intervalUnit; info.slidingUnit = pReq->slidingUnit; info.interval = pReq->interval; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 3a9e1d7405..246203f0c8 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -537,10 +537,12 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNode* pNode = NULL; FOREACH(pNode, pSortKeys) { SOrderByExprNode* pSortKey = (SOrderByExprNode*)pNode; + SExprNode* pSortExpr = (SExprNode*)pSortKey->pExpr; SNode* pTarget = NULL; bool found = false; FOREACH(pTarget, pTargets) { - if (0 == strcmp(((SExprNode*)pSortKey->pExpr)->aliasName, ((SColumnNode*)pTarget)->colName)) { + if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) || + (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))) { code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); if (TSDB_CODE_SUCCESS != code) { break; @@ -549,7 +551,7 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, } } if (TSDB_CODE_SUCCESS == code && !found) { - SNode* pCol = stbSplCreateColumnNode((SExprNode*)pSortKey->pExpr); + SNode* pCol = stbSplCreateColumnNode(pSortExpr); code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pCol)); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pTargets, pCol); diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp index b065ad0a59..2ca662cf86 100644 --- a/source/libs/planner/test/planOrderByTest.cpp +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -28,6 +28,8 @@ TEST_F(PlanOrderByTest, basic) { // ORDER BY key is not in the projection list run("SELECT c1 FROM t1 ORDER BY c2"); + run("SELECT c1 AS a FROM t1 ORDER BY a"); + run("SELECT c1 + 10 AS a FROM t1 ORDER BY a"); } @@ -59,4 +61,6 @@ TEST_F(PlanOrderByTest, stable) { run("SELECT c2 FROM st1 ORDER BY c1"); run("SELECT c2 FROM st1 PARTITION BY c2 ORDER BY c1"); + + run("SELECT c1 AS a FROM st1 ORDER BY a"); } From d320326371e3223fc2eb05489846aaad46b2ca60 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Jun 2022 14:41:17 +0800 Subject: [PATCH 82/90] reset query cache --- include/libs/catalog/catalog.h | 2 + source/client/src/clientImpl.c | 3 + source/client/src/clientMain.c | 3 + source/common/src/tglobal.c | 8 +- source/libs/catalog/inc/catalogInt.h | 34 +++--- source/libs/catalog/src/catalog.c | 31 ++++-- source/libs/catalog/src/ctgAsync.c | 4 +- source/libs/catalog/src/ctgCache.c | 155 ++++++++++++++++++-------- source/libs/catalog/src/ctgDbg.c | 20 ++-- source/libs/catalog/src/ctgRemote.c | 6 +- source/libs/catalog/src/ctgUtil.c | 2 +- source/libs/command/src/command.c | 4 +- source/libs/qworker/src/qworker.c | 4 +- source/libs/scheduler/src/schJob.c | 38 ++++--- source/libs/scheduler/src/schRemote.c | 6 +- source/libs/scheduler/src/schUtil.c | 2 +- source/libs/scheduler/src/scheduler.c | 20 +++- tools/shell/src/shellEngine.c | 3 +- 18 files changed, 227 insertions(+), 118 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f3600a509e..c0116051af 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -292,6 +292,8 @@ int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t reqId, bool forceUpdate); +int32_t catalogClearCache(void); + /** * Destroy catalog and relase all resources */ diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6f0f1c7f41..d04cc90ee4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -609,6 +609,9 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*)param; pRequest->code = code; + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, + pRequest->self, code, tstrerror(code), pRequest->requestId); + STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 2111ff4508..9a5d5bbeca 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -837,6 +837,9 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { SReqResultInfo *pResultInfo = &pRequest->body.resInfo; + tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, + pRequest->self, code, tstrerror(code), pRequest->requestId); + pResultInfo->pData = pResult; pResultInfo->numOfRows = 0; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2e1a539980..7457fe7eb6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -60,7 +60,7 @@ int32_t tsNumOfVnodeWriteThreads = 2; int32_t tsNumOfVnodeSyncThreads = 2; int32_t tsNumOfVnodeMergeThreads = 2; int32_t tsNumOfQnodeQueryThreads = 2; -int32_t tsNumOfQnodeFetchThreads = 2; +int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeUniqueThreads = 2; @@ -418,8 +418,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 1); if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 1, 1024, 0) != 0) return -1; - tsNumOfVnodeFetchThreads = tsNumOfCores / 2; - tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 2, 4); + tsNumOfVnodeFetchThreads = TRANGE(tsNumOfVnodeFetchThreads, 1, 1); if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfVnodeWriteThreads = tsNumOfCores; @@ -438,8 +437,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 1); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; - tsNumOfQnodeFetchThreads = tsNumOfCores / 2; - tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 2, 4); + tsNumOfQnodeFetchThreads = TRANGE(tsNumOfQnodeFetchThreads, 1, 1); if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfSnodeSharedThreads = tsNumOfCores / 4; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index df4b9b6ea6..7c90c3538c 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -59,6 +59,7 @@ enum { CTG_OP_UPDATE_VG_EPSET, CTG_OP_UPDATE_TB_INDEX, CTG_OP_DROP_TB_INDEX, + CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -328,6 +329,10 @@ typedef struct SCtgDropTbIndexMsg { char tbName[TSDB_TABLE_NAME_LEN]; } SCtgDropTbIndexMsg; +typedef struct SCtgClearCacheMsg { + SCatalog* pCtg; +} SCtgClearCacheMsg; + typedef struct SCtgUpdateEpsetMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -471,8 +476,8 @@ typedef struct SCtgOperation { #define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) -void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); -void ctgdShowClusterCache(SCatalog* pCtg); +void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p); +void ctgdShowClusterCache(SCatalog* pCtg); int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); @@ -487,8 +492,8 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *action); int32_t ctgOpUpdateUser(SCtgCacheOperation *action); int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); -void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); -void ctgRUnlockVgInfo(SCtgDBCache *dbCache); +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); +void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); @@ -502,17 +507,20 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp); +int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); -void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache); +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache); int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes); int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation); +int32_t ctgOpClearCache(SCtgCacheOperation *operation); + @@ -535,22 +543,22 @@ int32_t ctgMakeAsyncRes(SCtgJob *pJob); int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst); int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput); int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList); -void ctgFreeJob(void* job); -void ctgFreeHandle(SCatalog* pCtg); -void ctgFreeVgInfo(SDBVgInfo *vgInfo); +void ctgFreeJob(void* job); +void ctgFreeHandle(SCatalog* pCtg); +void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); -void ctgResetTbMetaTask(SCtgTask* pTask); -void ctgFreeDbCache(SCtgDBCache *dbCache); +void ctgResetTbMetaTask(SCtgTask* pTask); +void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); -void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); +void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); -char *ctgTaskTypeStr(CTG_TASK_TYPE type); +char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); -void ctgFreeSTableIndex(void *info); +void ctgFreeSTableIndex(void *info); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a3576f8738..bb02895569 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -105,7 +105,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* code = ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, &DbOut, NULL); if (code) { if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { - ctgDebug("db no longer exist, dbFName:%s, dbId:%" PRIx64, input.db, input.dbId); + ctgDebug("db no longer exist, dbFName:%s, dbId:0x%" PRIx64, input.db, input.dbId); ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId); } @@ -571,7 +571,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { } if (NULL == gCtgMgmt.pCluster) { - qError("catalog cluster cache are not ready, clusterId:%" PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:0x%" PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -583,7 +583,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { if (ctg && (*ctg)) { *catalogHandle = *ctg; - qDebug("got catalog handle from cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, *ctg); + qDebug("got catalog handle from cache, clusterId:0x%" PRIx64 ", CTG:%p", clusterId, *ctg); return TSDB_CODE_SUCCESS; } @@ -612,11 +612,11 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) { continue; } - qError("taosHashPut CTG to cache failed, clusterId:%" PRIx64, clusterId); + qError("taosHashPut CTG to cache failed, clusterId:0x%" PRIx64, clusterId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("add CTG to cache, clusterId:%" PRIx64 ", CTG:%p", clusterId, clusterCtg); + qDebug("add CTG to cache, clusterId:0x%" PRIx64 ", CTG:%p", clusterId, clusterCtg); break; } @@ -640,7 +640,7 @@ void catalogFreeHandle(SCatalog* pCtg) { } if (taosHashRemove(gCtgMgmt.pCluster, &pCtg->clusterId, sizeof(pCtg->clusterId))) { - ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:%" PRIx64, pCtg->clusterId); + ctgWarn("taosHashRemove from cluster failed, may already be freed, clusterId:0x%" PRIx64, pCtg->clusterId); return; } @@ -650,7 +650,7 @@ void catalogFreeHandle(SCatalog* pCtg) { ctgFreeHandle(pCtg); - ctgInfo("handle freed, culsterId:%" PRIx64, clusterId); + ctgInfo("handle freed, culsterId:0x%" PRIx64, clusterId); } int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* version, int64_t* dbId, int32_t* tableNum) { @@ -1247,6 +1247,23 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { CTG_API_LEAVE(ctgUpdateUserEnqueue(pCtg, pAuth, false)); } +int32_t catalogClearCache(void) { + CTG_API_ENTER(); + + qInfo("start to clear catalog cache"); + + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + int32_t code = ctgClearCacheEnqueue(NULL, true); + + qInfo("clear catalog cache end, code: %s", tstrerror(code)); + + CTG_API_LEAVE(code); +} + + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 8dabd56934..6adadf5045 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -622,7 +622,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { SCtgJob* pJob = pTask->pJob; int32_t code = 0; - qDebug("QID:0x%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); + qDebug("QID:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); pTask->code = rspCode; @@ -1276,7 +1276,7 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); - qDebug("QID:0x%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId); + qDebug("QID:0x%" PRIx64 " ctg start to launch task %d", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 9b4a774d8b..c70a5b1769 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -69,6 +69,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex + }, + { + CTG_OP_CLEAR_CACHE, + "clear cache", + ctgOpClearCache } }; @@ -81,7 +86,7 @@ int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { if (dbCache->deleted) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); *inCache = false; return TSDB_CODE_SUCCESS; @@ -92,7 +97,7 @@ int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); *inCache = false; - ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db vgInfo is empty, dbId:0x%"PRIx64, dbCache->dbId); return TSDB_CODE_SUCCESS; } @@ -105,7 +110,7 @@ int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_LOCK(CTG_WRITE, &dbCache->vgCache.vgLock); if (dbCache->deleted) { - ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); + ctgDebug("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } @@ -280,27 +285,27 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, int32_t sz = 0; char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { - ctgDebug("stb %" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); goto _return; } pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); if (NULL == pCache) { - ctgDebug("stb %" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); taosHashRelease(dbCache->stbCache, stName); goto _return; } CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { - ctgDebug("stb %" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); goto _return; } *pDb = dbCache; *pTb = pCache; - ctgDebug("stb %" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + ctgDebug("stb 0x%" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); CTG_CACHE_STAT_INC(tbMetaHitNum, 1); @@ -434,14 +439,14 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); - ctgDebug("stb %" PRIx64 " meta not in cache", ctx->tbInfo.suid); + ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); return TSDB_CODE_SUCCESS; } STableMeta* stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -492,7 +497,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, *sver = tbMeta->sversion; *tver = tbMeta->tversion; - ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:%" PRIx64, + ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:0x%" PRIx64, pTableName->tname, dbFName, *tbType, *sver, *tver, *suid); ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -507,14 +512,14 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("stb %" PRIx64 " meta not in cache", *suid); + ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } STableMeta* stbMeta = tbCache->pMeta; if (stbMeta->suid != *suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , stbMeta->suid, *suid); + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid:0x%" PRIx64 , stbMeta->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -990,6 +995,33 @@ _return: } +int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_CLEAR_CACHE; + op->syncOp = syncOp; + + SCtgClearCacheMsg *msg = taosMemoryMalloc(sizeof(SCtgClearCacheMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgClearCacheMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; @@ -1019,19 +1051,19 @@ int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) if (NULL == slot->meta) { slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); if (NULL == slot->meta) { - qError("taosArrayInit %d failed, id:%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); + qError("taosArrayInit %d failed, id:0x%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } if (NULL == taosArrayPush(slot->meta, meta)) { - qError("taosArrayPush meta to rent failed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("taosArrayPush meta to rent failed, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } slot->needSort = true; - qDebug("add meta to rent, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("add meta to rent, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1047,7 +1079,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1060,20 +1092,20 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qDebug("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta not found in slot, id:0x%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } memcpy(orig, meta, size); - qDebug("meta in rent updated, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent updated, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qDebug("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qDebug("meta in rent update failed, will try to add it, code:%x, id:0x%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1088,7 +1120,7 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1100,13 +1132,13 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); if (idx < 0) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("meta not found in slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } taosArrayRemove(slot->meta, idx); - qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent removed, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1219,11 +1251,11 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId); CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%"PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; @@ -1246,7 +1278,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { suid = taosHashGetKey(pIter, NULL); if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + ctgDebug("stb removed from rent, suid:0x%"PRIx64, *suid); } pIter = taosHashIterate(dbCache->stbCache, pIter); @@ -1257,7 +1289,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { uint64_t dbId = dbCache->dbId; - ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + ctgInfo("start to remove db from cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbCache->dbId); CTG_LOCK(CTG_WRITE, &dbCache->dbLock); @@ -1268,7 +1300,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgDebug("db removed from rent, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1276,7 +1308,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d } CTG_CACHE_STAT_DEC(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + ctgInfo("db removed from cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; } @@ -1339,7 +1371,7 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uin CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", + ctgDebug("db %s,0x%" PRIx64 " stb %s,0x%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); return TSDB_CODE_SUCCESS; @@ -1349,7 +1381,7 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uin int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { taosMemoryFree(meta); - ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + ctgError("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } @@ -1370,10 +1402,10 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam if (origType == TSDB_SUPER_TABLE) { if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%"PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%"PRIx64, dbFName, tbName, orig->suid); } origSuid = orig->suid; @@ -1407,13 +1439,13 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam } if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { - ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); + ctgError("taosHashPut to stable cache failed, suid:0x%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - ctgDebug("stb %" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); + ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); @@ -1424,7 +1456,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa if (NULL == dbCache->tbCache) { ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); - ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + ctgError("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } @@ -1510,7 +1542,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SCtgDBCache *dbCache = NULL; CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, msg->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%"PRIx64, dbFName, msg->dbId); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1540,7 +1572,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); ctgWUnlockVgInfo(dbCache); @@ -1569,7 +1601,7 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgInfo("dbId already updated, dbFName:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + ctgInfo("dbId already updated, dbFName:%s, dbId:0x%"PRIx64 ", targetId:0x%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); goto _return; } @@ -1629,7 +1661,7 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pMeta->dbFName, pMeta->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, pMeta->dbFName, pMeta->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pMeta->dbFName, pMeta->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1673,27 +1705,28 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { } if (msg->dbId && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", stb:%s, suid:%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); + ctgDebug("dbId already modified, dbFName:%s, current:0x%"PRIx64", dbId:0x%"PRIx64", stb:%s, suid:0x%"PRIx64, + msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); return TSDB_CODE_SUCCESS; } if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); } if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { - ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); _return: @@ -1714,7 +1747,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); + ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } @@ -1898,6 +1931,37 @@ _return: } +int32_t ctgOpClearCache(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgClearCacheMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + + if (pCtg) { + catalogFreeHandle(pCtg); + goto _return; + } + + void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + while (pIter) { + pCtg = *(SCatalog**)pIter; + + if (pCtg) { + catalogFreeHandle(pCtg); + } + + pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); + } + + taosHashClear(gCtgMgmt.pCluster); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + void ctgUpdateThreadUnexpectedStopped(void) { if (!atomic_load_8((int8_t*)&gCtgMgmt.exit) && CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); } @@ -1969,6 +2033,7 @@ void* ctgUpdateThreadFunc(void* param) { CTG_RT_STAT_INC(qDoneNum, 1); + ctgdShowCacheInfo(); ctgdShowClusterCache(pCtg); } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index ff93bedb21..7f2b919f17 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true}; +SCtgDebug gCTGDebug = {.cacheEnable = true}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); @@ -40,9 +40,9 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { STableComInfo *c = &p->tableInfo; if (TSDB_CHILD_TABLE == p->tableType) { - qDebug("table meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, p->tableType, p->vgId, p->uid, p->suid); + qDebug("table meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64, p->tableType, p->vgId, p->uid, p->suid); } else { - qDebug("table meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + qDebug("table meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); } @@ -75,7 +75,7 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { num = taosArrayGetSize(pResult->pDbInfo); for (int32_t i = 0; i < num; ++i) { SDbInfo *pDb = taosArrayGet(pResult->pDbInfo, i); - qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId); + qDebug("db %d dbInfo: vgVer:%d, tbNum:%d, dbId:0x%" PRIx64, i, pDb->vgVer, pDb->tbNum, pDb->dbId); } } else { qDebug("empty db info"); @@ -333,10 +333,10 @@ void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) { STableComInfo *c = &p->tableInfo; if (TSDB_CHILD_TABLE == p->tableType) { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid); return; } else { - ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", + ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:0x%" PRIx64 ",suid:0x%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d", tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize); } @@ -377,7 +377,7 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { } } - ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", + ctgDebug("[%d] db [%.*s][0x%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); pIter = taosHashIterate(dbHash, pIter); @@ -392,13 +392,13 @@ void ctgdShowClusterCache(SCatalog* pCtg) { return; } - ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg); + ctgDebug("## cluster 0x%"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg); ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); ctgdShowDBCache(pCtg, pCtg->dbCache); - ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg); + ctgDebug("## cluster 0x%"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg); } int32_t ctgdShowCacheInfo(void) { @@ -407,6 +407,8 @@ int32_t ctgdShowCacheInfo(void) { } CTG_API_ENTER(); + + qDebug("# total catalog cluster number %d #", taosHashGetSize(gCtgMgmt.pCluster)); SCatalog *pCtg = NULL; void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 178025704f..fa1a262832 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -186,13 +186,13 @@ int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode) SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); if (NULL == pJob) { - qDebug("job refId %" PRIx64 " already dropped", cbParam->refId); + qDebug("ctg job refId 0x%" PRIx64 " already dropped", cbParam->refId); goto _return; } SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId); - qDebug("QID:0x%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); + qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); @@ -263,7 +263,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTask* pTask CTG_ERR_JRET(code); } - ctgDebug("req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); + ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index e97c34dc26..476eb371b0 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -434,7 +434,7 @@ void ctgFreeJob(void* job) { taosMemoryFree(job); - qDebug("QID:%" PRIx64 ", job %" PRIx64 " freed", qid, rid); + qDebug("QID:0x%" PRIx64 ", ctg job 0x%" PRIx64 " freed", qid, rid); } int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target) { diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index f330b7ce16..f06664b60b 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -14,6 +14,7 @@ */ #include "command.h" +#include "catalog.h" #include "tdatablock.h" static int32_t getSchemaBytes(const SSchema* pSchema) { @@ -120,8 +121,7 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { } static int32_t execResetQueryCache() { - // todo - return TSDB_CODE_SUCCESS; + return catalogClearCache(); } int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 451607e7d0..db05b70658 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -614,6 +614,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + rsp = NULL; + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); } else { @@ -633,7 +635,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { rsp = NULL; qwMsg->connInfo = ctx->dataConnInfo; - qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code); + qwBuildAndSendFetchRsp(&qwMsg->connInfo, NULL, 0, code); QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index f9c7b21cf2..972ec4586c 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -21,9 +21,9 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { qDebug("acquire jobId:0x%"PRIx64, refId); return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { qDebug("sch acquire jobId:0x%"PRIx64, refId); return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJob(int64_t refId) { qDebug("release jobId:0x%"PRIx64, refId); return taosReleaseRef(schMgmt.jobRef, refId); } +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { qDebug("sch release jobId:0x%"PRIx64, refId); return taosReleaseRef(schMgmt.jobRef, refId); } int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) { pTask->plan = pPlan; @@ -47,7 +47,7 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("QID:%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -108,7 +108,7 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b atomic_add_fetch_32(&schMgmt.jobNum, 1); if (NULL == schAcquireJob(refId)) { - SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_JOB_ELOG("schAcquireJob job failed, refId:0x%" PRIx64, refId); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -194,7 +194,7 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { *pStatus = status; } - if (pJob->reqKilled) { + if (*pJob->reqKilled) { schUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED); @@ -229,7 +229,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_NOT_START: - if (newStatus != JOB_TASK_STATUS_EXECUTING) { + if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_DROPPING) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -299,7 +299,7 @@ int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync) { int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { - SCH_JOB_ELOG("job need to stop cause of status %s", jobTaskStatusStr(status)); + SCH_JOB_ELOG("abort op %s cause of job need to stop", schGetOpStr(type)); SCH_ERR_JRET(pJob->errCode); } @@ -308,7 +308,7 @@ int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync) { SCH_ERR_JRET(TSDB_CODE_TSC_APP_ERROR); } - SCH_JOB_ELOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); + SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); pJob->opStatus.sync = sync; @@ -377,7 +377,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("children info, the %d child TID %" PRIx64, n, (*childTask)->taskId); + SCH_TASK_DLOG("children info, the %d child TID 0x%" PRIx64, n, (*childTask)->taskId); } if (parentNum > 0) { @@ -411,7 +411,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("parents info, the %d parent TID %" PRIx64, n, (*parentTask)->taskId); + SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); } SCH_TASK_DLOG("level:%d, parentNum:%d, childNum:%d", i, parentNum, childNum); @@ -981,7 +981,9 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod schUpdateJobErrCode(pJob, errCode); int32_t code = atomic_load_32(&pJob->errCode); - SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); + if (code) { + SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); + } schPostJobRes(pJob, 0); @@ -1174,7 +1176,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_UNLOCK(SCH_WRITE, &parent->lock); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { - SCH_TASK_DLOG("all %d children task done, start to launch parent task %" PRIx64, readyNum, parent->taskId); + SCH_TASK_DLOG("all %d children task done, start to launch parent task 0x%" PRIx64, readyNum, parent->taskId); SCH_ERR_RET(schLaunchTask(pJob, parent)); } } @@ -1347,7 +1349,7 @@ int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTas int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { schGetTaskFromList(pJob->taskList, taskId, pTask); if (NULL == *pTask) { - SCH_JOB_ELOG("task not found in job task list, taskId:%" PRIx64, taskId); + SCH_JOB_ELOG("task not found in job task list, taskId:0x%" PRIx64, taskId); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -1520,9 +1522,9 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.queryRes); taosMemoryFreeClear(pJob->resData); - taosMemoryFreeClear(pJob); + taosMemoryFree(pJob); - qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); + qDebug("QID:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); if (jobNum == 0) { @@ -1614,7 +1616,7 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { int32_t code = 0; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { - qError("QID:%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); + qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; pReq->fp(NULL, pReq->cbParam, code); SCH_ERR_RET(code); @@ -1643,13 +1645,13 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { } if (NULL == schAcquireJob(refId)) { - SCH_JOB_ELOG("schAcquireJob job failed, refId:%" PRIx64, refId); + SCH_JOB_ELOG("schAcquireJob job failed, refId:0x%" PRIx64, refId); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } pJob->refId = refId; - SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId); + SCH_JOB_DLOG("job refId:0x%" PRIx64, pJob->refId); pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 2e9e5e5d1a..0bd747785c 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -344,7 +344,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } case TDMT_VND_DROP_TASK_RSP: { // SHOULD NEVER REACH HERE - SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:%" PRIx64, pJob->refId); + SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); break; } @@ -374,7 +374,7 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { - qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:%" PRIx64, + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 "taosAcquireRef job failed, may be dropped, refId:0x%" PRIx64, pParam->queryId, pParam->taskId, pParam->refId); SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); } @@ -443,7 +443,7 @@ int32_t schHandleExplainCallback(void *param, const SDataBuf *pMsg, int32_t code int32_t schHandleDropCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - qDebug("QID:%" PRIx64 ",TID:%" PRIx64 " drop task rsp received, code:%x", pParam->queryId, pParam->taskId, code); + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); taosMemoryFreeClear(param); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 93e1fda1d3..73077cbf0f 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -200,7 +200,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { SCH_UNLOCK(SCH_WRITE, &hb->lock); SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); - qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, + qDebug("hb connection updated, sId:0x%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->pTrans, trans->pHandle); return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 1f2780e1d7..57a405ffa3 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -62,12 +62,14 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { SCH_ERR_RET(TSDB_CODE_QRY_SYS_ERROR); } - qInfo("scheduler %" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); + qInfo("scheduler 0x%" PRIx64 " initizlized, maxJob:%u", schMgmt.sId, schMgmt.cfg.maxJobNum); return TSDB_CODE_SUCCESS; } int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) { + qDebug("scheduler sync exec job start"); + if (NULL == pReq || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -76,6 +78,8 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) } int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) { + qDebug("scheduler async exec job start"); + int32_t code = 0; if (NULL == pReq || NULL == pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -93,6 +97,8 @@ _return: } int32_t schedulerFetchRows(int64_t job, void **pData) { + qDebug("scheduler sync fetch rows start"); + if (NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -115,6 +121,8 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { } void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) { + qDebug("scheduler async fetch rows start"); + int32_t code = 0; if (NULL == fp || NULL == param) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); @@ -146,12 +154,12 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { int32_t code = 0; SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { - qDebug("acquire job from jobRef list failed, may not started or dropped, refId:%" PRIx64, job); + qDebug("acquire job from jobRef list failed, may not started or dropped, refId:0x%" PRIx64, job); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } if (pJob->status < JOB_TASK_STATUS_NOT_START || pJob->levelNum <= 0 || NULL == pJob->levels) { - qDebug("job not initialized or not executable job, refId:%" PRIx64, job); + qDebug("job not initialized or not executable job, refId:0x%" PRIx64, job); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -206,14 +214,14 @@ void schedulerFreeJob(int64_t job, int32_t errCode) { int32_t code = schProcessOnJobDropped(pJob, errCode); if (TSDB_CODE_SCH_JOB_IS_DROPPING == code) { - SCH_JOB_DLOG("sch job is already dropping, refId:%" PRIx64, job); + SCH_JOB_DLOG("sch job is already dropping, refId:0x%" PRIx64, job); return; } - SCH_JOB_DLOG("start to remove job from jobRef list, refId:%" PRIx64, job); + SCH_JOB_DLOG("start to remove job from jobRef list, refId:0x%" PRIx64, job); if (taosRemoveRef(schMgmt.jobRef, job)) { - SCH_JOB_ELOG("remove job from job list failed, refId:%" PRIx64, job); + SCH_JOB_ELOG("remove job from job list failed, refId:0x%" PRIx64, job); } schReleaseJob(job); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8ed0e9ddcf..1f29237d38 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -855,8 +855,7 @@ void shellGetGrantInfo() { if (code == TSDB_CODE_OPS_NOT_SUPPORT) { fprintf(stdout, "Server is Community Edition, %s\n\n", sinfo); } else { - fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", taos_errno(shell.conn), - taos_errstr(shell.conn)); + fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", code, taos_errstr(tres)); } return; } From 764d59b481a7dddbf292dba89fbfcb12aba9ce61 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 17 Jun 2022 14:56:54 +0800 Subject: [PATCH 83/90] test: fix sim udf test error --- source/dnode/mnode/impl/src/mndDnode.c | 3 ++- source/util/src/tenv.c | 3 +++ tests/script/sh/copy_udf.bat | 23 +++++++++++++++++++++++ tests/script/tsim/query/udf.sim | 10 ++++++++-- tests/tsim/src/simExe.c | 8 +++++++- 5 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 tests/script/sh/copy_udf.bat diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c936c0f93d..b631d4b631 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -776,11 +776,12 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false); - char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0}; + char *b = taosMemoryCalloc(VARSTR_HEADER_SIZE + strlen(offlineReason[pDnode->offlineReason]) + 1, 1); STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b, false); + taosMemoryFreeClear(b); numOfRows++; sdbRelease(pSdb, pDnode); diff --git a/source/util/src/tenv.c b/source/util/src/tenv.c index e717e82c5b..4fc0542816 100644 --- a/source/util/src/tenv.c +++ b/source/util/src/tenv.c @@ -72,6 +72,9 @@ int32_t taosEnvToCfg(const char *envStr, char *cfgStr) { if (cfgNameLen > 0) { memcpy(cfgStr, buf, cfgNameLen); memset(&cfgStr[cfgNameLen], ' ', p - cfgStr - cfgNameLen + 1); + } else { + *cfgStr = '\0'; + return -1; } } return strlen(cfgStr); diff --git a/tests/script/sh/copy_udf.bat b/tests/script/sh/copy_udf.bat new file mode 100644 index 0000000000..5144cf8d25 --- /dev/null +++ b/tests/script/sh/copy_udf.bat @@ -0,0 +1,23 @@ +@echo off + +echo Executing copy_udf.bat +set SCRIPT_DIR=%cd% +echo SCRIPT_DIR: %SCRIPT_DIR% + +cd ..\..\.. +set TAOS_DIR=%cd% +echo find udf library in %TAOS_DIR% +set UDF1_DIR=%TAOS_DIR%\debug\build\lib\udf1.dll +set UDF2_DIR=%TAOS_DIR%\debug\build\lib\udf2.dll + +echo %UDF1_DIR% +echo %UDF2_DIR% + +set UDF_TMP=C:\Windows\Temp\udf +rm -rf %UDF_TMP% +mkdir %UDF_TMP% + +echo Copy udf shared library files to %UDF_TMP% + +cp %UDF1_DIR% %UDF_TMP% +cp %UDF2_DIR% %UDF_TMP% diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 93cae4e391..d9821a0495 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -19,8 +19,14 @@ sql show databases; sql create table t (ts timestamp, f int); sql insert into t values(now, 1)(now+1s, 2); -sql create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8; -sql create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8; +system_content printf %OS% +if $system_content == Windows_NT then + sql create function udf1 as 'C:\\Windows\\Temp\\udf1.dll' outputtype int bufSize 8; + sql create aggregate function udf2 as 'C:\\Windows\\Temp\\udf2.dll' outputtype double bufSize 8; +else + sql create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8; + sql create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8; +endi sql show functions; if $rows != 2 then return -1 diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index dbc3c2c460..f97a13d2c5 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -458,11 +458,17 @@ bool simExecuteSystemContentCmd(SScript *script, char *option) { char buf[4096] = {0}; char buf1[4096 + 512] = {0}; char filename[400] = {0}; - sprintf(filename, "%s/%s.tmp", simScriptDir, script->fileName); + sprintf(filename, "%s" TD_DIRSEP "%s.tmp", simScriptDir, script->fileName); +#ifdef WINDOWS + sprintf(buf, "cd %s && ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + sprintf(buf1, "%s > %s 2>nul", buf, filename); +#else sprintf(buf, "cd %s; ", simScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); sprintf(buf1, "%s > %s 2>/dev/null", buf, filename); +#endif sprintf(script->system_exit_code, "%d", system(buf1)); simStoreSystemContentResult(script, filename); From d7660484cc5c32bbc6ce596ac3e9425ade32bc83 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 17 Jun 2022 15:10:00 +0800 Subject: [PATCH 84/90] fix(tmq): performance issue --- include/common/tcommon.h | 1 + include/util/taoserror.h | 3 ++- source/client/src/tmq.c | 23 +++++++++++++++++++++-- source/dnode/vnode/src/tq/tq.c | 1 + source/util/src/terror.c | 3 +++ 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 2e646f4769..a05287761e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -35,6 +35,7 @@ enum { TMQ_MSG_TYPE__DUMMY = 0, TMQ_MSG_TYPE__POLL_RSP, TMQ_MSG_TYPE__EP_RSP, + TMQ_MSG_TYPE__END_RSP, }; typedef enum EStreamType { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index fcbe5cd878..95acf55ec6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -429,7 +429,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) #define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) #define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) -#define TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0A0B) +#define TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0A0C) +#define TSDB_CODE_TQ_NO_COMMITTED_OFFSET TAOS_DEF_ERROR_CODE(0, 0x0A0D) // wal #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 9b58ddc589..41435e239e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -983,6 +983,19 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (code != 0) { tscWarn("msg discard from vg %d, epoch %d, code:%x", vgId, epoch, code); if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); + if (pRspWrapper == NULL) { + taosMemoryFree(pMsg->pData); + tscWarn("msg discard from vg %d, epoch %d since out of memory", vgId, epoch); + goto CREATE_MSG_FAIL; + } + pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; + /*pRspWrapper->vgHandle = pVg;*/ + /*pRspWrapper->topicHandle = pTopic;*/ + taosWriteQitem(tmq->mqueue, pRspWrapper); + tsem_post(&tmq->rspSem); + } goto CREATE_MSG_FAIL; } @@ -1115,7 +1128,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 0 +#if 1 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ bool set = false; @@ -1503,7 +1516,11 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (rspWrapper == NULL) return NULL; } - if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { + taosFreeQitem(rspWrapper); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; + return NULL; + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ int32_t consumerEpoch = atomic_load_32(&tmq->epoch); @@ -1564,6 +1581,8 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { return (TAOS_RES*)rspObj; + } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { + return NULL; } if (timeout != -1) { int64_t endTime = taosGetTimestampMs(); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c34738f083..70b09ec701 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -169,6 +169,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__NONE) { tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s", consumerId, pTq->pVnode->config.vgId, pReq->subKey); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return -1; } tqDebug("consumer %ld, restore offset of %s on vg %d failed, config is %ld, set to %ld", consumerId, pReq->subKey, diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 596aab0e09..009f217aa0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -572,6 +572,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in ca TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") +//tq +TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "No committed offset") + TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") From b864da2a9cf7adcdefd24731e4f2b7c5db9895f6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Jun 2022 15:26:45 +0800 Subject: [PATCH 85/90] fix app release issue --- include/common/tmsg.h | 2 +- source/common/src/tmsg.c | 4 ++-- source/dnode/mnode/impl/src/mndProfile.c | 17 +++++++++-------- source/libs/parser/src/parTranslater.c | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 23c9c1a632..ada11d90b2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1322,7 +1322,7 @@ int32_t tSerializeSKillQueryReq(void* buf, int32_t bufLen, SKillQueryReq* pReq); int32_t tDeserializeSKillQueryReq(void* buf, int32_t bufLen, SKillQueryReq* pReq); typedef struct { - int32_t connId; + uint32_t connId; } SKillConnReq; int32_t tSerializeSKillConnReq(void* buf, int32_t bufLen, SKillConnReq* pReq); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6bef6f08cd..d5dab72ef2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3432,7 +3432,7 @@ int32_t tSerializeSKillConnReq(void *buf, int32_t bufLen, SKillConnReq *pReq) { tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->connId) < 0) return -1; + if (tEncodeU32(&encoder, pReq->connId) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3445,7 +3445,7 @@ int32_t tDeserializeSKillConnReq(void *buf, int32_t bufLen, SKillConnReq *pReq) tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->connId) < 0) return -1; + if (tDecodeU32(&decoder, &pReq->connId) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 97a53e0eb8..fd80679316 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -76,7 +76,7 @@ int32_t mndInitProfile(SMnode *pMnode) { // in ms int32_t checkTime = tsShellActivityTimer * 2 * 1000; - pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_INT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + pMgmt->connCache = taosCacheInit(TSDB_DATA_TYPE_UINT, checkTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->connCache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to alloc profile cache since %s", terrstr()); @@ -124,7 +124,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType char connStr[255] = {0}; int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); - int32_t connId = mndGenerateUid(connStr, len); + uint32_t connId = mndGenerateUid(connStr, len); if (startTime == 0) startTime = taosGetTimestampMs(); SConnObj connObj = {.id = connId, @@ -145,7 +145,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); int32_t keepTime = tsShellActivityTimer * 3; - SConnObj *pConn = taosCachePut(pMgmt->connCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); + SConnObj *pConn = taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); @@ -448,6 +448,7 @@ static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnIn return -1; } else { mDebug("a new app %" PRIx64 "created", pReq->appId); + mndReleaseApp(pMnode, pApp); return TSDB_CODE_SUCCESS; } } @@ -463,7 +464,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; - SRpcConnInfo connInfo = pMsg->conn; + SRpcConnInfo connInfo = pMsg->info.conn; mndUpdateAppInfo(pMnode, pHbReq, &connInfo); @@ -654,7 +655,7 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->conn.user); + mInfo("connId:%x, queryId:%" PRIx64 " is killed by user:%s", connId, queryId, pReq->info.conn.user); pConn->killId = queryId; taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); return 0; @@ -680,13 +681,13 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { return -1; } - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(int32_t)); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); if (pConn == NULL) { - mError("connId:%d, failed to kill connection, conn not exist", killReq.connId); + mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, is killed by user:%s", killReq.connId, pReq->info.conn.user); + mInfo("connId:%u, is killed by user:%s", killReq.connId, pReq->info.conn.user); pConn->killed = 1; taosCacheRelease(pMgmt->connCache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 57c464f09c..09d4bfab14 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3890,7 +3890,7 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) { SKillConnReq killReq = {0}; killReq.connId = pStmt->targetId; - return buildCmdMsg(pCxt, TDMT_MND_KILL_CONN, (FSerializeFunc)tSerializeSKillQueryReq, &killReq); + return buildCmdMsg(pCxt, TDMT_MND_KILL_CONN, (FSerializeFunc)tSerializeSKillConnReq, &killReq); } static int32_t translateKillQuery(STranslateContext* pCxt, SKillQueryStmt* pStmt) { From 105a6e0892a10d79083e3f6d02c8f5135f49f0c9 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 17 Jun 2022 15:37:05 +0800 Subject: [PATCH 86/90] test: fix sim user login error --- tests/tsim/src/simParse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 638c4a1ccb..5b6dda4dae 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -206,7 +206,7 @@ SScript *simParseScript(char *fileName) { for (int32_t i = 0; i < cmdlen; ++i) { if (buffer[i] == '\r' || buffer[i] == '\n') { - buffer[i] = ' '; + buffer[i] = '\0'; } } From 61153b76378d4837d1665feef7ed0c1e2cbb5c81 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 17 Jun 2022 14:13:45 +0800 Subject: [PATCH 87/90] feat(stream): distribute stream session plan --- include/libs/nodes/nodes.h | 1 + include/libs/nodes/plannodes.h | 12 ++++-- source/libs/executor/inc/executorimpl.h | 6 ++- source/libs/executor/src/executorimpl.c | 19 ++++------ source/libs/executor/src/timewindowoperator.c | 38 +++++++++---------- source/libs/nodes/src/nodesCloneFuncs.c | 11 +++++- source/libs/nodes/src/nodesCodeFuncs.c | 8 ++++ source/libs/nodes/src/nodesUtilFuncs.c | 5 +++ source/libs/planner/src/planLogicCreater.c | 3 +- source/libs/planner/src/planPhysiCreater.c | 17 ++++++--- source/libs/planner/src/planSpliter.c | 31 ++++++++++++--- 11 files changed, 103 insertions(+), 48 deletions(-) diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3620f45408..64f65ddea8 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -227,6 +227,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, + QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 749d58b224..a9002b5d19 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -130,13 +130,17 @@ typedef struct SMergeLogicNode { typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; -typedef enum EIntervalAlgorithm { +typedef enum EWindowAlgorithm { INTERVAL_ALGO_HASH = 1, INTERVAL_ALGO_MERGE, INTERVAL_ALGO_STREAM_FINAL, INTERVAL_ALGO_STREAM_SEMI, INTERVAL_ALGO_STREAM_SINGLE, -} EIntervalAlgorithm; + SESSION_ALGO_STREAM_SEMI, + SESSION_ALGO_STREAM_FINAL, + SESSION_ALGO_STREAM_SINGLE, + SESSION_ALGO_MERGE, +} EWindowAlgorithm; typedef struct SWindowLogicNode { SLogicNode node; @@ -153,7 +157,7 @@ typedef struct SWindowLogicNode { int8_t triggerType; int64_t watermark; double filesFactor; - EIntervalAlgorithm intervalAlgo; + EWindowAlgorithm windowAlgo; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -371,6 +375,8 @@ typedef struct SSessionWinodwPhysiNode { } SSessionWinodwPhysiNode; typedef SSessionWinodwPhysiNode SStreamSessionWinodwPhysiNode; +typedef SSessionWinodwPhysiNode SStreamSemiSessionWinodwPhysiNode; +typedef SSessionWinodwPhysiNode SStreamFinalSessionWinodwPhysiNode; typedef struct SStateWinodwPhysiNode { SWinodwPhysiNode window; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 034e2893df..d9a7dc8718 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -845,8 +845,10 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, + SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, + SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 40b019eb5d..b8516a3056 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4728,18 +4728,13 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { - SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - - STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType}; - - SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; - - pOptr = createStreamSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, tsSlotId, &as, - pTaskInfo); - + pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { + int32_t children = 0; + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { + int32_t children = 1; + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d7ae823522..d827067f1c 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2425,12 +2425,15 @@ int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, Sql return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo)); } -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { + SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + int32_t tsSlotId = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + int32_t code = TSDB_CODE_OUT_OF_MEMORY; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -2453,12 +2456,14 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx } initDummyFunction(pInfo->pDummyCtx, pInfo->binfo.pCtx, numOfCols); - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = (STimeWindowAggSupp) {.waterMark = pSessionNode->window.watermark, + .calTrigger = pSessionNode->window.triggerType, + .maxTs = INT64_MIN}; initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->primaryTsIndex = tsSlotId; - pInfo->gap = gap; + pInfo->gap = pSessionNode->gap; pInfo->binfo.pRes = pResBlock; pInfo->order = TSDB_ORDER_ASC; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -2960,25 +2965,20 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, - int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, - SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - SStreamSessionAggOperatorInfo* pInfo = NULL; - SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pExprInfo, numOfCols, pResBlock, gap, - tsSlotId, pTwAggSupp, pTaskInfo); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, + SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo); if (pOperator == NULL) { goto _error; } pOperator->name = "StreamFinalSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION; - int32_t numOfChild = 1; // Todo(liuyao) get it from phy plan - pInfo = pOperator->info; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; pInfo->pChildren = taosArrayInit(8, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChild = - createStreamSessionAggOperatorInfo(NULL, pExprInfo, numOfCols, NULL, gap, tsSlotId, pTwAggSupp, pTaskInfo); + createStreamSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo); if (pChild == NULL) { goto _error; } @@ -2988,7 +2988,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream _error: if (pInfo != NULL) { - destroyStreamSessionAggOperatorInfo(pInfo, numOfCols); + destroyStreamSessionAggOperatorInfo(pInfo, pOperator->numOfExprs); } taosMemoryFreeClear(pInfo); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2e30b01357..d0eb1548c1 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -427,7 +427,7 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(filesFactor); - COPY_SCALAR_FIELD(intervalAlgo); + COPY_SCALAR_FIELD(windowAlgo); return (SNode*)pDst; } @@ -538,6 +538,12 @@ static SNode* physiIntervalCopy(const SIntervalPhysiNode* pSrc, SIntervalPhysiNo return (SNode*)pDst; } +static SNode* physiSessionCopy(const SSessionWinodwPhysiNode* pSrc, SSessionWinodwPhysiNode* pDst) { + COPY_BASE_OBJECT_FIELD(window, physiWindowCopy); + COPY_SCALAR_FIELD(gap); + return (SNode*)pDst; +} + static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) { COPY_SCALAR_FIELD(dataBlockId); CLONE_NODE_LIST_FIELD(pSlots); @@ -678,6 +684,9 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return physiIntervalCopy((const SIntervalPhysiNode*)pNode, (SIntervalPhysiNode*)pDst); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + return physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst); default: break; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index fb6a428d3c..ff433660be 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -246,6 +246,10 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: return "PhysiStreamSessionWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + return "PhysiStreamSemiSessionWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + return "PhysiStreamFinalSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: return "PhysiStateWindow"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: @@ -3998,6 +4002,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiFillNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: @@ -4131,6 +4137,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiFillNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index a45ba53ad1..719171df18 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -287,6 +287,10 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: return makeNode(type, sizeof(SStreamSessionWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + return makeNode(type, sizeof(SStreamSemiSessionWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + return makeNode(type, sizeof(SStreamFinalSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: return makeNode(type, sizeof(SStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: @@ -804,6 +808,7 @@ void nodesDestroyNode(SNode* pNode) { } case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index cbc74a2711..3cd5eeb655 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -548,6 +548,7 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->winType = WINDOW_TYPE_SESSION; pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; + pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol); if (NULL == pWindow->pTspk) { @@ -572,7 +573,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); - pWindow->intervalAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; + pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 2974b3ef8c..99476c4cbd 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -977,8 +977,8 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* return code; } -static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { - switch (intervalAlgo) { +static ENodeType getIntervalOperatorType(EWindowAlgorithm windowAlgo) { + switch (windowAlgo) { case INTERVAL_ALGO_HASH: return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; case INTERVAL_ALGO_MERGE: @@ -989,6 +989,14 @@ static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { return QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; case INTERVAL_ALGO_STREAM_SINGLE: return QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; + case SESSION_ALGO_STREAM_FINAL: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION; + case SESSION_ALGO_STREAM_SEMI: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION; + case SESSION_ALGO_STREAM_SINGLE: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + case SESSION_ALGO_MERGE: + return QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION; default: break; } @@ -998,7 +1006,7 @@ static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode( - pCxt, (SLogicNode*)pWindowLogicNode, getIntervalOperatorType(pWindowLogicNode->intervalAlgo)); + pCxt, (SLogicNode*)pWindowLogicNode, getIntervalOperatorType(pWindowLogicNode->windowAlgo)); if (NULL == pInterval) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1015,8 +1023,7 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode( - pCxt, (SLogicNode*)pWindowLogicNode, - (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION : QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION)); + pCxt, (SLogicNode*)pWindowLogicNode, getIntervalOperatorType(pWindowLogicNode->windowAlgo)); if (NULL == pSession) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 3a9e1d7405..f79d64e190 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -376,8 +376,8 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo SLogicNode* pPartWindow = NULL; int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); if (TSDB_CODE_SUCCESS == code) { - ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH; - ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_MERGE; + ((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_HASH; + ((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_MERGE; SNodeList* pMergeKeys = NULL; code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { @@ -400,8 +400,8 @@ static int32_t stbSplSplitIntervalForStream(SSplitContext* pCxt, SStableSplitInf SLogicNode* pPartWindow = NULL; int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); if (TSDB_CODE_SUCCESS == code) { - ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_STREAM_SEMI; - ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_STREAM_FINAL; + ((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_STREAM_SEMI; + ((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_STREAM_FINAL; code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); } if (TSDB_CODE_SUCCESS == code) { @@ -421,8 +421,29 @@ static int32_t stbSplSplitInterval(SSplitContext* pCxt, SStableSplitInfo* pInfo) } } +static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pPartWindow = NULL; + int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); + if (TSDB_CODE_SUCCESS == code) { + ((SWindowLogicNode*)pPartWindow)->windowAlgo = SESSION_ALGO_STREAM_SEMI; + ((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = SESSION_ALGO_STREAM_FINAL; + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + (SNode*)splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); + return code; +} + static int32_t stbSplSplitSession(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - return TSDB_CODE_PLAN_INTERNAL_ERROR; + if (pCxt->pPlanCxt->streamQuery) { + return stbSplSplitSessionForStream(pCxt, pInfo); + } else { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } } static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { From e92aa57e93d61ecb3dfde072a6db5b453cc8ba35 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 17 Jun 2022 16:38:06 +0800 Subject: [PATCH 88/90] feat(tmq): support commit one msg --- examples/c/tmq.c | 30 +++--- include/client/taos.h | 30 +++--- include/util/taoserror.h | 2 + source/client/src/tmq.c | 211 ++++++++++++++++++++++++++++++--------- source/util/src/terror.c | 2 + tests/test/c/tmqDemo.c | 6 +- tests/test/c/tmqSim.c | 14 +-- 7 files changed, 205 insertions(+), 90 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 6b09e8c658..7870d7d9a1 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -146,8 +146,8 @@ int32_t create_topic() { return 0; } -void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { - printf("commit %d tmq %p offsets %p param %p\n", resp, tmq, offsets, param); +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); } tmq_t* build_consumer() { @@ -183,10 +183,10 @@ tmq_list_t* build_topic_list() { } void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { - tmq_resp_err_t err; + int32_t code; - if ((err = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); printf("subscribe err\n"); return; } @@ -205,9 +205,9 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } } - err = tmq_consumer_close(tmq); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); else fprintf(stderr, "%% Consumer closed\n"); } @@ -215,11 +215,11 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { static const int MIN_COMMIT_COUNT = 1; - int msg_count = 0; - tmq_resp_err_t err; + int msg_count = 0; + int32_t code; - if ((err = tmq_subscribe(tmq, topics))) { - fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); return; } @@ -245,9 +245,9 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } } - err = tmq_consumer_close(tmq); - if (err) - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); else fprintf(stderr, "%% Consumer closed\n"); } diff --git a/include/client/taos.h b/include/client/taos.h index 1510f38074..e9f7f88387 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -209,21 +209,20 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi /* --------------------------TMQ INTERFACE------------------------------- */ +#if 0 enum { TMQ_RESP_ERR__FAIL = -1, TMQ_RESP_ERR__SUCCESS = 0, }; typedef int32_t tmq_resp_err_t; +#endif -typedef struct tmq_t tmq_t; -typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; -typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; - +typedef struct tmq_t tmq_t; typedef struct tmq_conf_t tmq_conf_t; typedef struct tmq_list_t tmq_list_t; -typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); +typedef void(tmq_commit_cb(tmq_t *, int32_t code, void *param)); DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); @@ -233,24 +232,19 @@ DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *); DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); -DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); +DLL_EXPORT const char *tmq_err2str(int32_t code); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ -DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); -DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); -DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); +DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); +DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq); +DLL_EXPORT int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); // timeout: -1 means infinitely waiting -DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); -DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); +DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); +DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); -DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); -DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); - -#if 0 -DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); -DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset); -#endif +DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); +DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a5668de381..ea624aa32b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -710,6 +710,8 @@ int32_t* taosGetErrno(); //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) +//tmq +#define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #ifdef __cplusplus } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 41435e239e..638b4f1ea5 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -48,14 +48,6 @@ struct tmq_list_t { SArray container; }; -struct tmq_topic_vgroup_t { - SMqOffset offset; -}; - -struct tmq_topic_vgroup_list_t { - SArray container; // SArray -}; - struct tmq_conf_t { char clientId[256]; char groupId[TSDB_CGROUP_LEN]; @@ -161,9 +153,9 @@ typedef struct { } SMqPollRspWrapper; typedef struct { - tmq_t* tmq; - tsem_t rspSem; - tmq_resp_err_t rspErr; + tmq_t* tmq; + tsem_t rspSem; + int32_t rspErr; } SMqSubscribeCbParam; typedef struct { @@ -189,7 +181,7 @@ typedef struct { int8_t freeOffsets; tmq_commit_cb* userCb; tsem_t rspSem; - tmq_resp_err_t rspErr; + int32_t rspErr; SArray* offsets; void* userParam; } SMqCommitCbParam; @@ -201,7 +193,7 @@ typedef struct { int8_t freeOffsets; int32_t waitingRspNum; int32_t totalRspNum; - tmq_resp_err_t rspErr; + int32_t rspErr; tmq_commit_cb* userCb; /*SArray* successfulOffsets;*/ /*SArray* failedOffsets;*/ @@ -347,10 +339,9 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { pParam->rspErr = code; if (pParam->async) { if (pParam->automatic && pParam->tmq->commitCb) { - pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets, - pParam->tmq->commitCbUserParam); + pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, pParam->tmq->commitCbUserParam); } else if (!pParam->automatic && pParam->userCb) { - pParam->userCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets, pParam->userParam); + pParam->userCb(pParam->tmq, pParam->rspErr, pParam->userParam); } if (pParam->freeOffsets) { @@ -388,10 +379,10 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { if (pParamSet->async) { // call async cb func if (pParamSet->automatic && pParamSet->tmq->commitCb) { - pParamSet->tmq->commitCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->tmq->commitCbUserParam); + pParamSet->tmq->commitCb(pParamSet->tmq, pParamSet->rspErr, pParamSet->tmq->commitCbUserParam); } else if (!pParamSet->automatic && pParamSet->userCb) { // sem post - pParamSet->userCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->userParam); + pParamSet->userCb(pParamSet->tmq, pParamSet->rspErr, pParamSet->userParam); } } else { tsem_post(&pParamSet->rspSem); @@ -405,10 +396,132 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { return 0; } -int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async, - tmq_commit_cb* userCb, void* userParam) { +int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; + if (msg != NULL) { + SMqRspObj* pRspObj = (SMqRspObj*)msg; + if (!TD_RES_TMQ(pRspObj)) { + return TSDB_CODE_TMQ_INVALID_MSG; + } + + SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); + if (pParamSet == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pParamSet->tmq = tmq; + pParamSet->automatic = automatic; + pParamSet->async = async; + pParamSet->freeOffsets = 1; + pParamSet->userCb = userCb; + pParamSet->userParam = userParam; + tsem_init(&pParamSet->rspSem, 0, 0); + + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(pTopic->topicName, pRspObj->topic) == 0) { + for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->vgId == pRspObj->vgId) { + if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { + tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); + + return 0; + } + + STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + if (pOffset == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pOffset->type = TMQ_OFFSET__LOG; + pOffset->version = pVg->currentOffset; + + int32_t groupLen = strlen(tmq->groupId); + memcpy(pOffset->subKey, tmq->groupId, groupLen); + pOffset->subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + if (code < 0) { + ASSERT(0); + } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSTqOffset(&encoder, pOffset); + + // build param + SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + pParam->params = pParamSet; + pParam->pOffset = pOffset; + + // build send info + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pMsgSendInfo == NULL) { + // TODO + continue; + } + pMsgSendInfo->msgInfo = (SDataBuf){ + .pData = buf, + .len = sizeof(SMsgHead) + len, + .handle = NULL, + }; + + tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, + pVg->vgId, pOffset->version); + + // TODO: put into cb + pVg->committedOffset = pVg->currentOffset; + + pMsgSendInfo->requestId = generateRequestId(); + pMsgSendInfo->requestObjRefId = 0; + pMsgSendInfo->param = pParam; + pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; + // send msg + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); + pParamSet->waitingRspNum++; + pParamSet->totalRspNum++; + } + } + } + } + if (pParamSet->totalRspNum == 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); + return 0; + } + + if (!async) { + tsem_wait(&pParamSet->rspSem); + code = pParamSet->rspErr; + tsem_destroy(&pParamSet->rspSem); + } else { + code = 0; + } + + if (code != 0 && async) { + if (automatic) { + tmq->commitCb(tmq, code, tmq->commitCbUserParam); + } else { + userCb(tmq, code, userParam); + } + } + return 0; + } + SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -521,9 +634,9 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 if (code != 0 && async) { if (automatic) { - tmq->commitCb(tmq, code, NULL, tmq->commitCbUserParam); + tmq->commitCb(tmq, code, tmq->commitCbUserParam); } else { - userCb(tmq, code, NULL, userParam); + userCb(tmq, code, userParam); } } @@ -537,7 +650,8 @@ int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8 return 0; } -int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async, +#if 0 +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { SMqCMCommitOffsetReq req; SArray* pOffsets = NULL; @@ -547,7 +661,7 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_ int8_t freeOffsets; int32_t code = -1; - if (offsets == NULL) { + if (msg == NULL) { freeOffsets = 1; pOffsets = taosArrayInit(0, sizeof(SMqOffset)); for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -564,7 +678,7 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_ } } else { freeOffsets = 0; - pOffsets = (SArray*)&offsets->container; + pOffsets = (SArray*)&msg->container; } req.num = (int32_t)pOffsets->size; @@ -651,6 +765,7 @@ END: } return code; } +#endif void tmqAssignDelayedHbTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; @@ -729,7 +844,7 @@ int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { +int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { if (*topics == NULL) { *topics = tmq_list_new(); } @@ -737,12 +852,12 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { SMqClientTopic* topic = taosArrayGet(tmq->clientTopics, i); tmq_list_append(*topics, strchr(topic->topicName, '.') + 1); } - return TMQ_RESP_ERR__SUCCESS; + return 0; } -tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { - tmq_list_t* lst = tmq_list_new(); - tmq_resp_err_t rsp = tmq_subscribe(tmq, lst); +int32_t tmq_unsubscribe(tmq_t* tmq) { + tmq_list_t* lst = tmq_list_new(); + int32_t rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); return rsp; } @@ -858,11 +973,13 @@ FAIL: return NULL; } -tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { +#if 0 +int32_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { return tmqCommitInner2(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); } +#endif -tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { +int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -902,7 +1019,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { if (sendInfo == NULL) goto FAIL; SMqSubscribeCbParam param = { - .rspErr = TMQ_RESP_ERR__SUCCESS, + .rspErr = 0, .tmq = tmq, }; @@ -1337,7 +1454,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return code; } -tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { +#if 0 +int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { const SMqOffset* pOffset = &offset->offset; if (strcmp(pOffset->cgroup, tmq->groupId) != 0) { return TMQ_RESP_ERR__FAIL; @@ -1359,6 +1477,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { } return TMQ_RESP_ERR__FAIL; } +#endif SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { int64_t reqOffset; @@ -1599,10 +1718,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } } -tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { +int32_t tmq_consumer_close(tmq_t* tmq) { if (tmq->status == TMQ_CONSUMER_STATUS__READY) { - tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL); - if (rsp != TMQ_RESP_ERR__SUCCESS) { + int32_t rsp = tmq_commit_sync(tmq, NULL); + if (rsp != 0) { return rsp; } @@ -1610,18 +1729,18 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); - if (rsp != TMQ_RESP_ERR__SUCCESS) { + if (rsp != 0) { return rsp; } } // TODO: free resources - return TMQ_RESP_ERR__SUCCESS; + return 0; } -const char* tmq_err2str(tmq_resp_err_t err) { - if (err == TMQ_RESP_ERR__SUCCESS) { +const char* tmq_err2str(int32_t err) { + if (err == 0) { return "success"; - } else if (err == TMQ_RESP_ERR__FAIL) { + } else if (err == -1) { return "fail"; } else { return tstrerror(err); @@ -1667,10 +1786,8 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -void tmq_commit_async(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, tmq_commit_cb* cb, void* param) { - tmqCommitInner2(tmq, offsets, 0, 1, cb, param); +void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { + tmqCommitInner2(tmq, msg, 0, 1, cb, param); } -tmq_resp_err_t tmq_commit_sync(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { - return tmqCommitInner2(tmq, offsets, 0, 0, NULL, NULL); -} +int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 753ff643fd..0c1e796fa4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -583,6 +583,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "No committed offset TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") + #ifdef TAOS_ERROR_C }; #endif diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 5a972e071a..02fd3c1396 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -353,8 +353,8 @@ tmq_list_t* build_topic_list() { void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { static const int MIN_COMMIT_COUNT = 1000; - int msg_count = 0; - tmq_resp_err_t err; + int msg_count = 0; + int32_t err; if ((err = tmq_subscribe(tmq, topics))) { fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); @@ -379,7 +379,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLogSize) { - tmq_resp_err_t err; + int32_t err; if ((err = tmq_subscribe(tmq, topics))) { fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 3737484a2c..0f78a003d6 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -339,8 +339,8 @@ int queryDB(TAOS* taos, char* command) { return 0; } -static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) { - pError("tmq_commit_cb_print() commit %d\n", resp); +static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + pError("tmq_commit_cb_print() commit %d\n", code); } void build_consumer(SThreadInfo* pInfo) { @@ -443,7 +443,7 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { } void loop_consume(SThreadInfo* pInfo) { - tmq_resp_err_t err; + int32_t code; int64_t totalMsgs = 0; int64_t totalRows = 0; @@ -496,8 +496,8 @@ void* consumeThreadFunc(void* param) { return NULL; } - tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); - if (err) { + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } @@ -517,14 +517,14 @@ void* consumeThreadFunc(void* param) { } err = tmq_unsubscribe(pInfo->tmq); - if (err) { + if (err != 0) { pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); /*pInfo->consumeMsgCnt = -1;*/ /*return NULL;*/ } err = tmq_consumer_close(pInfo->tmq); - if (err) { + if (err != 0) { pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); /*exit(-1);*/ } From 8ce76f741962fd76f71ab9838c73e67ab2294ad9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 17 Jun 2022 17:30:57 +0800 Subject: [PATCH 89/90] fix case issues --- source/libs/command/src/explain.c | 6 ++++++ source/libs/planner/src/planOptimizer.c | 1 + source/libs/scheduler/src/schJob.c | 5 ++++- tests/script/tsim/db/alter_option.sim | 2 +- tests/script/tsim/db/basic6.sim | 4 ++-- tests/script/tsim/db/create_all_options.sim | 2 +- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 79861dfa05..8f91282480 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -194,6 +194,11 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = fillPhysiNode->node.pChildren; break; } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: { + STableMergeScanPhysiNode *mergePhysiNode = (STableMergeScanPhysiNode *)pNode; + pPhysiChildren = mergePhysiNode->scan.node.pChildren; + break; + } default: qError("not supported physical node type %d", pNode->type); QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -398,6 +403,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_TBL_SCAN_FORMAT, pTblScanNode->scan.tableName.tname); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9d7cd0cf27..a7c25162b7 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -731,6 +731,7 @@ static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNo FOREACH(pNode, pSort->node.pParent->pChildren) { if (nodesEqualNode(pNode, (SNode*)pSort)) { REPLACE_NODE(pDownNode); + ((SLogicNode*)pDownNode)->pParent = pSort->node.pParent; break; } } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 972ec4586c..733f8694cc 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -116,7 +116,7 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b SCH_JOB_DLOG("job refId:0x%" PRIx64" created", pJob->refId); - pJob->status = JOB_TASK_STATUS_NOT_START; + schUpdateJobStatus(pJob, JOB_TASK_STATUS_NOT_START); *pSchJob = pJob; @@ -1623,12 +1623,15 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { } pJob->sql = pReq->sql; + pJob->reqKilled = pReq->reqKilled; pJob->attr.queryJob = true; pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->queryId = pReq->pDag->queryId; pJob->subPlans = pReq->pDag->pSubplans; pJob->userRes.execFp = pReq->fp; pJob->userRes.userParam = pReq->cbParam; + + schUpdateJobStatus(pJob, JOB_TASK_STATUS_NOT_START); code = schBeginOperation(pJob, SCH_OP_EXEC, sync); if (code) { diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 4351ee5cb1..fede960a6a 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -92,7 +92,7 @@ endi if $data5_db != no_strict then # strict return -1 endi -if $data6_db != 345600 then # duration +if $data6_db != 345600m then # duration return -1 endi if $data7_db != 1440000m,1440000m,1440000m then # keep diff --git a/tests/script/tsim/db/basic6.sim b/tests/script/tsim/db/basic6.sim index 64103b5dac..7525fe2087 100644 --- a/tests/script/tsim/db/basic6.sim +++ b/tests/script/tsim/db/basic6.sim @@ -34,7 +34,7 @@ endi if $data24 != 1 then return -1 endi -if $data26 != 2880 then +if $data26 != 2880m then return -1 endi if $data27 != 14400m,14400m,14400m then @@ -78,7 +78,7 @@ endi if $data24 != 1 then return -1 endi -if $data26 != 21600 then +if $data26 != 21600m then return -1 endi diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 284875ee08..efe7ff99cf 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -113,7 +113,7 @@ endi if $data5_db != no_strict then # strict return -1 endi -if $data6_db != 14400 then # duration +if $data6_db != 14400m then # duration return -1 endi if $data7_db != 5256000m,5256000m,5256000m then # keep From 73e4dd9543bc3427f2f56d038ae1268159dc94c5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 17 Jun 2022 17:35:14 +0800 Subject: [PATCH 90/90] test: fix tmq case possibly fails --- tests/system-test/7-tmq/subscribeDb1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index ed92a429ae..c08c7d3dae 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -442,7 +442,7 @@ class TDTestCase: showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - time.sleep(3) + time.sleep(6) tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): os.system("TASKKILL /F /IM tmq_sim.exe")