From 0a83ecac8150ae69ce2ae1eff7b7a8aeb88ee9e8 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 4 Jan 2022 10:40:34 +0800 Subject: [PATCH 01/54] add sub structure --- include/common/tmsg.h | 104 ++++++++++++---- include/util/tdef.h | 1 + source/dnode/mnode/impl/inc/mndDef.h | 103 +++++++++++----- source/dnode/mnode/impl/src/mndConsumer.c | 140 +++++++++++++++++----- 4 files changed, 264 insertions(+), 84 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ba8bb7ca2f..2b17e6db7a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -320,14 +320,23 @@ typedef struct SEpSet { } SEpSet; static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { - if(buf == NULL) return sizeof(SEpSet); - memcpy(buf, pEp, sizeof(SEpSet)); - //TODO: endian conversion - return sizeof(SEpSet); + int tlen = 0; + tlen += taosEncodeFixedI8(buf, pEp->inUse); + tlen += taosEncodeFixedI8(buf, pEp->numOfEps); + for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + tlen += taosEncodeFixedU16(buf, pEp->port[i]); + tlen += taosEncodeString(buf, pEp->fqdn[i]); + } + return tlen; } -static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEpSet) { - memcpy(pEpSet, buf, sizeof(SEpSet)); +static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { + buf = taosDecodeFixedI8(buf, &pEp->inUse); + buf = taosDecodeFixedI8(buf, &pEp->numOfEps); + for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + buf = taosDecodeFixedU16(buf, &pEp->port[i]); + buf = taosDecodeStringTo(buf, pEp->fqdn[i]); + } return buf; } @@ -1088,16 +1097,16 @@ typedef struct STaskDropRsp { } STaskDropRsp; typedef struct { - int8_t igExists; - char* name; - char* physicalPlan; - char* logicalPlan; + int8_t igExists; + char* name; + char* physicalPlan; + char* logicalPlan; } SCMCreateTopicReq; static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) { int tlen = 0; - tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedI8(buf, pReq->igExists); + tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeString(buf, pReq->physicalPlan); tlen += taosEncodeString(buf, pReq->logicalPlan); return tlen; @@ -1127,41 +1136,62 @@ static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopi } typedef struct { - char* topicName; - char* consumerGroup; + int32_t topicNum; int64_t consumerId; + char* consumerGroup; + char* topicName[]; } SCMSubscribeReq; static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { int tlen = 0; - tlen += taosEncodeString(buf, pReq->topicName); - tlen += taosEncodeString(buf, pReq->consumerGroup); + tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); + tlen += taosEncodeString(buf, pReq->consumerGroup); + for(int i = 0; i < pReq->topicNum; i++) { + tlen += taosEncodeString(buf, pReq->topicName[i]); + } return tlen; } static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { - buf = taosDecodeString(buf, &pReq->topicName); - buf = taosDecodeString(buf, &pReq->consumerGroup); + buf = taosDecodeFixedI32(buf, &pReq->topicNum); buf = taosDecodeFixedI64(buf, &pReq->consumerId); + buf = taosDecodeString(buf, &pReq->consumerGroup); + for(int i = 0; i < pReq->topicNum; i++) { + buf = taosDecodeString(buf, &pReq->topicName[i]); + } return buf; } -typedef struct { +typedef struct SMqSubTopic { int32_t vgId; - SEpSet pEpSet; + int64_t topicId; + SEpSet epSet; +} SMqSubTopic; + +typedef struct { + int32_t topicNum; + SMqSubTopic topics[]; } SCMSubscribeRsp; static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { int tlen = 0; - tlen += taosEncodeFixedI32(buf, pRsp->vgId); - tlen += taosEncodeSEpSet(buf, &pRsp->pEpSet); + tlen += taosEncodeFixedI32(buf, pRsp->topicNum); + for(int i = 0; i < pRsp->topicNum; i++) { + tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId); + tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId); + tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet); + } return tlen; } static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { - buf = taosDecodeFixedI32(buf, &pRsp->vgId); - buf = taosDecodeSEpSet(buf, &pRsp->pEpSet); + buf = taosDecodeFixedI32(buf, &pRsp->topicNum); + for(int i = 0; i < pRsp->topicNum; i++) { + buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId); + buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId); + buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet); + } return buf; } @@ -1170,10 +1200,36 @@ typedef struct { int64_t consumerId; int64_t consumerGroupId; int64_t offset; + char* sql; + char* logicalPlan; + char* physicalPlan; } SMVSubscribeReq; +static FORCE_INLINE int tSerializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) { + int tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->topicId); + tlen += taosEncodeFixedI64(buf, pReq->consumerId); + tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId); + tlen += taosEncodeFixedI64(buf, pReq->offset); + tlen += taosEncodeString(buf, pReq->sql); + tlen += taosEncodeString(buf, pReq->logicalPlan); + tlen += taosEncodeString(buf, pReq->physicalPlan); + return tlen; +} + +static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->topicId); + buf = taosDecodeFixedI64(buf, &pReq->consumerId); + buf = taosDecodeFixedI64(buf, &pReq->consumerGroupId); + buf = taosDecodeFixedI64(buf, &pReq->offset); + buf = taosDecodeString(buf, &pReq->sql); + buf = taosDecodeString(buf, &pReq->logicalPlan); + buf = taosDecodeString(buf, &pReq->physicalPlan); + return buf; +} + typedef struct { - int64_t newOffset; + int64_t status; } SMVSubscribeRsp; typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index 233e9f0f55..9f16b58e0d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -177,6 +177,7 @@ do { \ #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_CONSUMER_GROUP_LEN 192 #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a874e67210..986914a8e2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -22,6 +22,7 @@ #include "sync.h" #include "tmsg.h" #include "thash.h" +#include "tlist.h" #include "tlog.h" #include "trpc.h" #include "ttimer.h" @@ -288,14 +289,81 @@ typedef struct { char payload[]; } SShowObj; +typedef struct SConsumerObj { + uint64_t uid; + int64_t createTime; + int64_t updateTime; + //uint64_t dbUid; + int32_t version; + SRWLatch lock; + SList* topics; +} SConsumerObj; + +typedef struct SMqSubConsumerObj { + int64_t consumerUid; // if -1, unassigned + SList* vgId; //SList +} SMqSubConsumerObj; + +typedef struct SMqSubCGroupObj { + char name[TSDB_CONSUMER_GROUP_LEN]; + SList* consumers; //SList +} SMqSubCGroupObj; + +typedef struct SMqSubTopicObj { + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + SRWLatch lock; + int32_t sqlLen; + char* sql; + char* logicalPlan; + char* physicalPlan; + SList* cgroups; //SList +} SMqSubTopicObj; + +typedef struct SMqConsumerSubObj { + int64_t topicUid; + SList* vgIds; //SList +} SMqConsumerSubObj; + +typedef struct SMqConsumerHbObj { + int64_t consumerId; + SList* consumerSubs; //SList +} SMqConsumerHbObj; + +typedef struct SMqVGroupSubObj { + int64_t topicUid; + SList* consumerIds; //SList +} SMqVGroupSubObj; + +typedef struct SMqVGroupHbObj { + int64_t vgId; + SList* vgSubs; //SList +} SMqVGroupHbObj; + +typedef struct SCGroupObj { + char name[TSDB_TOPIC_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + uint64_t uid; + //uint64_t dbUid; + int32_t version; + SRWLatch lock; + SList* consumerIds; +} SCGroupObj; + typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; uint64_t uid; uint64_t dbUid; - int32_t version; + int32_t version; SRWLatch lock; int32_t execLen; void* executor; @@ -303,32 +371,9 @@ typedef struct { char* sql; char* logicalPlan; char* physicalPlan; + SList* consumerIds; } STopicObj; -typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - uint64_t uid; - //uint64_t dbUid; - int32_t version; - SRWLatch lock; - -} SConsumerObj; - -typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - uint64_t uid; - //uint64_t dbUid; - int32_t version; - SRWLatch lock; - -} SCGroupObj; - typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; char db[TSDB_DB_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 9e7cdbf09e..5391b0f73e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -66,17 +66,82 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} +static SSdbRaw *mndCGroupActionEncode(SCGroupObj *pCGroup) { + int32_t size = sizeof(SConsumerObj) + MND_CONSUMER_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pCGroup->name, TSDB_TABLE_FNAME_LEN); + SDB_SET_INT64(pRaw, dataPos, pCGroup->createTime); + SDB_SET_INT64(pRaw, dataPos, pCGroup->updateTime); + SDB_SET_INT64(pRaw, dataPos, pCGroup->uid); + /*SDB_SET_INT64(pRaw, dataPos, pConsumer->dbUid);*/ + SDB_SET_INT32(pRaw, dataPos, pCGroup->version); + + int32_t sz = listNEles(pCGroup->consumerIds); + SDB_SET_INT32(pRaw, dataPos, sz); + + SListIter iter; + tdListInitIter(pCGroup->consumerIds, &iter, TD_LIST_FORWARD); + SListNode *pn = NULL; + while ((pn = tdListNext(&iter)) != NULL) { + int64_t consumerId = *(int64_t *)pn->data; + SDB_SET_INT64(pRaw, dataPos, consumerId); + } + + SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE); + SDB_SET_DATALEN(pRaw, dataPos); + return pRaw; +} + +static SSdbRow *mndCGroupActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != MND_CONSUMER_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode cgroup since %s", terrstr()); + return NULL; + } + + // TODO: maximum size is not known + int32_t size = sizeof(SCGroupObj) + 128 * sizeof(int64_t); + SSdbRow *pRow = sdbAllocRow(size); + SCGroupObj *pCGroup = sdbGetRowObj(pRow); + if (pCGroup == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pCGroup->name, TSDB_TABLE_FNAME_LEN); + SDB_GET_INT64(pRaw, pRow, dataPos, &pCGroup->createTime); + SDB_GET_INT64(pRaw, pRow, dataPos, &pCGroup->updateTime); + SDB_GET_INT64(pRaw, pRow, dataPos, &pCGroup->uid); + /*SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->dbUid);*/ + SDB_GET_INT32(pRaw, pRow, dataPos, &pCGroup->version); + + int32_t sz; + SDB_GET_INT32(pRaw, pRow, dataPos, &sz); + // TODO: free list when failing + tdListInit(pCGroup->consumerIds, sizeof(int64_t)); + for (int i = 0; i < sz; i++) { + int64_t consumerId; + SDB_GET_INT64(pRaw, pRow, dataPos, &consumerId); + tdListAppend(pCGroup->consumerIds, &consumerId); + } + + SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_CONSUMER_RESERVE_SIZE); + return pRow; +} + static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer) { int32_t size = sizeof(SConsumerObj) + MND_CONSUMER_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) return NULL; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pConsumer->name, TSDB_TABLE_FNAME_LEN); - SDB_SET_BINARY(pRaw, dataPos, pConsumer->db, TSDB_DB_FNAME_LEN); + SDB_SET_INT64(pRaw, dataPos, pConsumer->uid); SDB_SET_INT64(pRaw, dataPos, pConsumer->createTime); SDB_SET_INT64(pRaw, dataPos, pConsumer->updateTime); - SDB_SET_INT64(pRaw, dataPos, pConsumer->uid); /*SDB_SET_INT64(pRaw, dataPos, pConsumer->dbUid);*/ SDB_SET_INT32(pRaw, dataPos, pConsumer->version); @@ -102,11 +167,9 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { if (pConsumer == NULL) return NULL; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumer->name, TSDB_TABLE_FNAME_LEN); - SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumer->db, TSDB_DB_FNAME_LEN); + SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->uid); SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->createTime); SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->updateTime); - SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->uid); /*SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->dbUid);*/ SDB_GET_INT32(pRaw, pRow, dataPos, &pConsumer->version); @@ -116,17 +179,17 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { } static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer) { - mTrace("consumer:%s, perform insert action", pConsumer->name); + mTrace("consumer:%ld, perform insert action", pConsumer->uid); return 0; } static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer) { - mTrace("consumer:%s, perform delete action", pConsumer->name); + mTrace("consumer:%ld, perform delete action", pConsumer->uid); return 0; } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pOldConsumer, SConsumerObj *pNewConsumer) { - mTrace("consumer:%s, perform update action", pOldConsumer->name); + mTrace("consumer:%ld, perform update action", pOldConsumer->uid); atomic_exchange_32(&pOldConsumer->updateTime, pNewConsumer->updateTime); atomic_exchange_32(&pOldConsumer->version, pNewConsumer->version); @@ -157,9 +220,34 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { char *msgStr = pMsg->rpcMsg.pCont; SCMSubscribeReq *pSubscribe; tDeserializeSCMSubscribeReq(msgStr, pSubscribe); - // add consumerGroupId -> list to sdb - // add consumerId -> list to sdb - // add consumer -> list to sdb + int topicNum = pSubscribe->topicNum; + int64_t consumerId = pSubscribe->consumerId; + char *consumerGroup = pSubscribe->consumerGroup; + // get consumer group and add client into it + SCGroupObj *pCGroup = sdbAcquire(pMnode->pSdb, SDB_CGROUP, consumerGroup); + if (pCGroup != NULL) { + // iterate the list until finding the consumer + // add consumer to cgroup list if not found + // put new record + } + + SConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + if (pConsumer != NULL) { + //reset topic list + } + + for (int i = 0; i < topicNum; i++) { + char *topicName = pSubscribe->topicName[i]; + STopicObj *pTopic = mndAcquireTopic(pMnode, topicName); + //get + // consumer id + SList *list = pTopic->consumerIds; + // add the consumer if not in the list + // + SList* topicList = pConsumer->topics; + //add to topic + } + return 0; } @@ -255,9 +343,7 @@ static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumO pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); if (pIter == NULL) break; - if (strcmp(pConsumer->db, dbName) == 0) { - numOfConsumers++; - } + numOfConsumers++; sdbRelease(pSdb, pConsumer); } @@ -316,11 +402,11 @@ static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMs return 0; } -static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { +static int32_t mndRetrieveCGroup(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; - SConsumerObj *pConsumer = NULL; + SCGroupObj *pCGroup = NULL; int32_t cols = 0; char *pWrite; char prefix[64] = {0}; @@ -330,36 +416,28 @@ static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer); + pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pCGroup); if (pShow->pIter == NULL) break; - if (strncmp(pConsumer->name, prefix, prefixLen) != 0) { - sdbRelease(pSdb, pConsumer); + if (strncmp(pCGroup->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pCGroup); continue; } cols = 0; char consumerName[TSDB_TABLE_NAME_LEN] = {0}; - tstrncpy(consumerName, pConsumer->name + prefixLen, TSDB_TABLE_NAME_LEN); + tstrncpy(consumerName, pCGroup->name + prefixLen, TSDB_TABLE_NAME_LEN); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, consumerName); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pConsumer->createTime; + *(int64_t *)pWrite = pCGroup->createTime; cols++; - /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ - /**(int32_t *)pWrite = pConsumer->numOfColumns;*/ - /*cols++;*/ - - /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ - /**(int32_t *)pWrite = pConsumer->numOfTags;*/ - /*cols++;*/ - numOfRows++; - sdbRelease(pSdb, pConsumer); + sdbRelease(pSdb, pCGroup); } pShow->numOfReads += numOfRows; From 1c7c7ee6720b73b9a5e55e200b28a32a8e512ac5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 4 Jan 2022 09:54:38 +0000 Subject: [PATCH 02/54] add more plugins --- .devcontainer/devcontainer.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 65d0b1915c..b258a2a252 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,13 +6,17 @@ "dockerfile": "Dockerfile", // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon - "args": { "VARIANT": "ubuntu-21.04" } + "args": { + "VARIANT": "ubuntu-21.04" + } }, - "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined" + ], // Set *default* container specific settings.json values on container create. "settings": {}, - // Add the IDs of extensions you want installed when the container is created. "extensions": [ "ms-vscode.cpptools", @@ -21,15 +25,13 @@ "visualstudioexptteam.vscodeintel", "eamodio.gitlens", "matepek.vscode-catch2-test-adapter", - "spmeesseman.vscode-taskexplorer" + "spmeesseman.vscode-taskexplorer", + "cschlosser.doxdocgen" ], - // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], - // Use 'postCreateCommand' to run commands after the container is created. // "postCreateCommand": "gcc -v", - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "root" -} +} \ No newline at end of file From cee5a2ecbe92212ee1b3bba2b39069973055069a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 18:01:31 +0800 Subject: [PATCH 03/54] optimize tfile --- include/util/tfile.h | 2 +- .../index/inc/index_fst_counting_writer.h | 5 +++++ .../index/src/index_fst_counting_writer.c | 20 ++++++++++++++++++- source/libs/index/test/indexTests.cc | 2 +- source/util/src/tfile.c | 10 ++++++++++ 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/include/util/tfile.h b/include/util/tfile.h index b3d141c443..d3813051a4 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -43,7 +43,7 @@ int32_t tfFsync(int64_t tfd); bool tfValid(int64_t tfd); int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); int32_t tfFtruncate(int64_t tfd, int64_t length); - +void * tfMmapReadOnly(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index fcc0d5a0b3..d7363f2f4c 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -22,6 +22,8 @@ extern "C" { #include "tfile.h" +//#define USE_MMAP 1 + #define DefaultMem 1024 * 1024 static char tmpFile[] = "./index"; @@ -39,6 +41,9 @@ typedef struct WriterCtx { bool readOnly; char buf[256]; int size; +#ifdef USE_MMAP + char* ptr; +#endif } file; struct { int32_t capa; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index c3e1aab381..2b64d65e46 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -31,7 +31,12 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { int nRead = 0; if (ctx->type == TFile) { +#ifdef USE_MMAP + nRead = len < ctx->file.size ? len : ctx->file.size; + memcpy(buf, ctx->file.ptr, nRead); +#else nRead = tfRead(ctx->file.fd, buf, len); +#endif } else { memcpy(buf, ctx->mem.buf + ctx->offset, len); } @@ -43,7 +48,13 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off int nRead = 0; if (ctx->type == TFile) { // tfLseek(ctx->file.fd, offset, 0); +#ifdef USE_MMAP + int32_t last = ctx->file.size - offset; + nRead = last >= len ? len : last; + memcpy(buf, ctx->file.ptr + offset, nRead); +#else nRead = tfPread(ctx->file.fd, buf, len, offset); +#endif } else { // refactor later assert(0); @@ -83,6 +94,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; +#ifdef USE_MMAP + ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); +#endif } memcpy(ctx->file.buf, path, strlen(path)); if (ctx->file.fd < 0) { @@ -111,8 +125,12 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { - // ctx->flush(ctx); tfClose(ctx->file.fd); + if (ctx->file.readOnly) { +#ifdef USE_MMAP + munmap(ctx->file.ptr, ctx->file.size); +#endif + } if (remove) { unlink(ctx->file.buf); } } free(ctx); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index c3d6e5541f..badc510b19 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -28,7 +28,7 @@ #include "tutil.h" using namespace std; -#define NUM_OF_THREAD 10 +#define NUM_OF_THREAD 5 class DebugInfo { public: diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 4cb20802c7..0f68e9204d 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -158,3 +158,13 @@ int32_t tfFtruncate(int64_t tfd, int64_t length) { taosReleaseRef(tsFileRsetId, tfd); return code; } + +void *tfMmapReadOnly(int64_t tfd, int64_t length) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return NULL; + int32_t fd = (int32_t)(uintptr_t)p; + + void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); + taosReleaseRef(tsFileRsetId, tfd); + return ptr; +} From 8279e49cb0ed6d9de62106095aff00f1280db9fb Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 20:18:15 +0800 Subject: [PATCH 04/54] optimize read performace case --- source/libs/index/test/indexTests.cc | 76 ++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index badc510b19..ef41855c33 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -679,6 +679,17 @@ class IndexObj { } return numOfTable; } + int ReadMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world", + size_t numOfTable = 100 * 10000) { + std::string tColVal = colVal; + + int colValSize = tColVal.size(); + for (int i = 0; i < numOfTable; i++) { + tColVal[i % colValSize] = 'a' + i % 26; + SearchOne(colName, tColVal); + } + return 0; + } int Put(SIndexMultiTerm* fvs, uint64_t uid) { numOfWrite += taosArrayGetSize(fvs); @@ -701,8 +712,8 @@ class IndexObj { int64_t s = taosGetTimestampUs(); if (Search(mq, result) == 0) { int64_t e = taosGetTimestampUs(); - std::cout << "search one successfully and time cost:" << e - s << "\tquery col:" << colName - << "\t val: " << colVal << "\t size:" << taosArrayGetSize(result) << std::endl; + std::cout << "search and time cost:" << e - s << "\tquery col:" << colName << "\t val: " << colVal + << "\t size:" << taosArrayGetSize(result) << std::endl; } else { } int sz = taosArrayGetSize(result); @@ -711,6 +722,31 @@ class IndexObj { return sz; // assert(taosArrayGetSize(result) == targetSize); } + int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + indexMultiTermQueryAdd(mq, term, QUERY_TERM); + + SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); + + int64_t s = taosGetTimestampUs(); + if (Search(mq, result) == 0) { + int64_t e = taosGetTimestampUs(); + std::cout << "search one successfully and time cost:" << e - s << "\tquery col:" << colName + << "\t val: " << colVal << "\t size:" << taosArrayGetSize(result) << std::endl; + } else { + } + int sz = taosArrayGetSize(result); + indexMultiTermQueryDestroy(mq); + taosArrayDestroy(result); + assert(sz == 1); + uint64_t* ret = (uint64_t*)taosArrayGet(result, 0); + assert(val = *ret); + + return sz; + } + void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), @@ -831,12 +867,15 @@ TEST_F(IndexEnv2, testIndex_TrigeFlush) { assert(numOfTable == target); } -static void write_and_search(IndexObj* idx) { - std::string colName("tag1"), colVal("Hello"); - +static void single_write_and_search(IndexObj* idx) { int target = idx->SearchOne("tag1", "Hello"); target = idx->SearchOne("tag2", "Test"); - // idx->PutOne(colName, colVal); +} +static void multi_write_and_search(IndexObj* idx) { + int target = idx->SearchOne("tag1", "Hello"); + target = idx->SearchOne("tag2", "Test"); + idx->WriteMultiMillonData("tag1", "Hello", 100 * 10000); + idx->WriteMultiMillonData("tag2", "Test", 100 * 10000); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { std::string path = "/tmp/cache_and_tfile"; @@ -851,7 +890,21 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { for (int i = 0; i < NUM_OF_THREAD; i++) { // - threads[i] = std::thread(write_and_search, index); + threads[i] = std::thread(single_write_and_search, index); + } + for (int i = 0; i < NUM_OF_THREAD; i++) { + // TOD + threads[i].join(); + } +} +TEST_F(IndexEnv2, testIndex_MultiWrite_and_MultiRead) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) {} + + std::thread threads[NUM_OF_THREAD]; + for (int i = 0; i < NUM_OF_THREAD; i++) { + // + threads[i] = std::thread(multi_write_and_search, index); } for (int i = 0; i < NUM_OF_THREAD; i++) { // TOD @@ -860,13 +913,16 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } TEST_F(IndexEnv2, testIndex_restart) { - std::string path = "/tmp/test1"; + std::string path = "/tmp/cache_and_tfile"; if (index->Init(path) != 0) {} + index->SearchOneTarget("tag1", "Hello", 10); + index->SearchOneTarget("tag2", "Test", 10); } -TEST_F(IndexEnv2, testIndex_performance) { - std::string path = "/tmp/test2"; +TEST_F(IndexEnv2, testIndex_read_performance) { + std::string path = "/tmp/cache_and_tfile"; if (index->Init(path) != 0) {} + index->ReadMultiMillonData("tag1", "Hello"); } TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/test3"; From 99379fd5a61c0ab536d8b43e701b697c86b5ed13 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 20:56:55 +0800 Subject: [PATCH 05/54] add test case --- source/libs/index/test/indexTests.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index ef41855c33..0ce48bcec4 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -755,6 +755,14 @@ class IndexObj { Put(terms, 10); indexMultiTermDestroy(terms); } + void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { + SIndexMultiTerm* terms = indexMultiTermCreate(); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + indexMultiTermAdd(terms, term); + Put(terms, val); + indexMultiTermDestroy(terms); + } void Debug() { std::cout << "numOfWrite:" << numOfWrite << std::endl; std::cout << "numOfRead:" << numOfRead << std::endl; @@ -922,7 +930,10 @@ TEST_F(IndexEnv2, testIndex_restart) { TEST_F(IndexEnv2, testIndex_read_performance) { std::string path = "/tmp/cache_and_tfile"; if (index->Init(path) != 0) {} + index->PutOneTarge("tag1", "Hello", 12); + index->PutOneTarge("tag1", "Hello", 15); index->ReadMultiMillonData("tag1", "Hello"); + std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; } TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/test3"; From 49ef94fb6972b3fd1d5409c5399f41d3fea9292f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 22:58:52 +0800 Subject: [PATCH 06/54] refactor code --- source/libs/index/src/index.c | 3 ++- source/libs/index/test/indexTests.cc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 44d3066589..b5a65a9fda 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -401,7 +401,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); - if (pReader == NULL) { indexWarn("empty pReader found"); } + if (pReader == NULL) { indexWarn("empty tfile reader found"); } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); @@ -512,6 +512,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { return ret; END: tfileWriterClose(tw); + return -1; } int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 0ce48bcec4..2ba6b505c5 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -934,6 +934,7 @@ TEST_F(IndexEnv2, testIndex_read_performance) { index->PutOneTarge("tag1", "Hello", 15); index->ReadMultiMillonData("tag1", "Hello"); std::cout << "reader sz: " << index->SearchOne("tag1", "Hello") << std::endl; + assert(3 == index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndexMultiTag) { std::string path = "/tmp/test3"; From a75899024c8d04f2966b12b6fa9dc98bd4168a66 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 23:09:30 +0800 Subject: [PATCH 07/54] add test case --- source/libs/index/test/indexTests.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 2ba6b505c5..9c92af26a2 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -937,6 +937,17 @@ TEST_F(IndexEnv2, testIndex_read_performance) { assert(3 == index->SearchOne("tag1", "Hello")); } TEST_F(IndexEnv2, testIndexMultiTag) { - std::string path = "/tmp/test3"; + std::string path = "/tmp/multi_tag"; if (index->Init(path) != 0) {} + index->WriteMultiMillonData("tag1", "Hello", 100 * 10000); + index->WriteMultiMillonData("tag2", "Test", 100 * 10000); + index->WriteMultiMillonData("tag3", "Test", 100 * 10000); + index->WriteMultiMillonData("tag4", "Test", 100 * 10000); +} +TEST_F(IndexEnv2, testLongComVal) { + std::string path = "/tmp/long_colVal"; + if (index->Init(path) != 0) {} + // gen colVal by randstr + std::string randstr = "xxxxxxxxxxxxxxxxx"; + index->WriteMultiMillonData("tag1", randstr, 100 * 10000); } From 82745dea8062c9ef15f22ba85aa6d4b485c5a0a3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 23:13:08 +0800 Subject: [PATCH 08/54] refactor code --- include/os/os.h | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index 53a6cef96a..0135865461 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,11 +22,13 @@ extern "C" { #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -36,18 +38,14 @@ extern "C" { #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include #include #include -#include +#include #include +#include +#include + +#include #include "osAtomic.h" #include "osDef.h" From a84af557023f947cf2708aa90435a27e85b68fd1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 4 Jan 2022 23:24:37 +0800 Subject: [PATCH 09/54] refactor code --- include/os/os.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/os.h b/include/os/os.h index 0135865461..972880da9c 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -40,7 +40,7 @@ extern "C" { #include #include #include -#include +#include #include #include #include From feee87ddfa14be2267a6035af257ea04adef76f9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 5 Jan 2022 10:11:26 +0800 Subject: [PATCH 10/54] feature/qnode --- source/libs/scheduler/inc/schedulerInt.h | 5 ++-- source/libs/scheduler/src/scheduler.c | 31 +++++++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index fa4ae0d152..1d5d0599e6 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -114,8 +114,9 @@ typedef struct SSchJob { #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) -#define SCH_JOB_ERR_LOG(param, ...) qError("QID:%"PRIx64 param, job->queryId, __VA_ARGS__) -#define SCH_TASK_ERR_LOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) +#define SCH_JOB_ELOG(param, ...) qError("QID:% "PRIx64 param, job->queryId, __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:% "PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:% "PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 22556f742b..ff254da6fb 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -270,6 +270,8 @@ int32_t schPushTaskToExecList(SSchJob *job, SSchTask *task) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("push to %s list", "execTasks"); + return TSDB_CODE_SUCCESS; } @@ -284,6 +286,8 @@ int32_t schMoveTaskToSuccList(SSchJob *job, SSchTask *task, bool *moved) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("push to %s list", "succTasks"); + *moved = true; return TSDB_CODE_SUCCESS; @@ -299,6 +303,8 @@ int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("push to %s list", "failTasks"); + *moved = true; return TSDB_CODE_SUCCESS; @@ -372,7 +378,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *job, SSchTask *task) { SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved)); if (!moved) { - SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); + SCH_TASK_ELOG("task may already moved, status:%d", task->status); return TSDB_CODE_SUCCESS; } @@ -447,11 +453,11 @@ int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) { SCH_ERR_RET(schTaskCheckAndSetRetry(job, task, errCode, &needRetry)); if (!needRetry) { - SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); + SCH_TASK_ELOG("task failed[%x], no more retry", errCode); SCH_ERR_RET(schMoveTaskToFailList(job, task, &moved)); if (!moved) { - SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); + SCH_TASK_ELOG("task may already moved, status:%d", task->status); } if (SCH_TASK_NEED_WAIT_ALL(task)) { @@ -492,6 +498,7 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms goto _task_error; } } + break; } case TDMT_VND_SUBMIT_RSP: { if (rspCode != TSDB_CODE_SUCCESS) { @@ -567,19 +574,25 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); - if (NULL == job || NULL == (*job)) { + SSchJob **pjob = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); + if (NULL == pjob || NULL == (*pjob)) { qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SSchTask **task = taosHashGet((*job)->execTasks, &pParam->taskId, sizeof(pParam->taskId)); - if (NULL == task || NULL == (*task)) { + SSchJob *job = *pjob; + + SSchTask **ptask = taosHashGet(job->execTasks, &pParam->taskId, sizeof(pParam->taskId)); + if (NULL == ptask || NULL == (*ptask)) { qError("taosHashGet taskId:%"PRIx64" not exist", pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } + + SSchTask *task = *ptask; + + SCH_TASK_DLOG("Got msg:%d, rspCode:%d", msgType, rspCode); - schProcessRspMsg(*job, *task, msgType, pMsg->pData, pMsg->len, rspCode); + schProcessRspMsg(job, task, msgType, pMsg->pData, pMsg->len, rspCode); _return: tfree(param); @@ -809,7 +822,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) { SCH_ERR_RET(schSetTaskCondidateAddrs(job, task)); if (NULL == task->condidateAddrs || taosArrayGetSize(task->condidateAddrs) <= 0) { - SCH_TASK_ERR_LOG("no valid condidate node for task:%"PRIx64, task->taskId); + SCH_TASK_ELOG("no valid condidate node for task:%"PRIx64, task->taskId); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } From 75077c4ae9078a06ab6d4e9158356c16135113d1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 5 Jan 2022 02:12:50 +0000 Subject: [PATCH 11/54] make multi-thread query work --- source/dnode/vnode/meta/src/metaBDBImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index ae6693f973..4220a604b4 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -255,7 +255,7 @@ static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path) { return -1; } - ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_MPOOL, 0); + ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_CDB | DB_INIT_MPOOL, 0); if (ret != 0) { BDB_PERR("Failed to open META env", ret); return -1; From ad965dbcb462508d5c2b3822510cb84a5603b982 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 4 Jan 2022 19:24:28 -0800 Subject: [PATCH 12/54] invalid release --- source/dnode/mnode/impl/src/mndBnode.c | 12 +++++++----- source/dnode/mnode/impl/src/mndQnode.c | 12 +++++++----- source/dnode/mnode/impl/src/mndSnode.c | 12 +++++++----- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 886784805e..312438a535 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -26,7 +26,7 @@ static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj); static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); -static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode); +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg); static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg); static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg); @@ -155,9 +155,9 @@ static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { return 0; } -static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode) { - mTrace("bnode:%d, perform update action, old_row:%p new_row:%p", pOldBnode->id, pOldBnode, pNewBnode); - pOldBnode->updateTime = pNewBnode->updateTime; +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew) { + mTrace("bnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + pOld->updateTime = pNew->updateTime; return 0; } @@ -251,6 +251,7 @@ static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { SBnodeObj *pObj = mndAcquireBnode(pMnode, pCreate->dnodeId); if (pObj != NULL) { mError("bnode:%d, bnode already exist", pObj->id); + terrno = TSDB_CODE_MND_BNODE_ALREADY_EXIST; mndReleaseBnode(pMnode, pObj); return -1; } @@ -370,11 +371,12 @@ static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { int32_t code = mndDropBnode(pMnode, pMsg, pObj); if (code != 0) { + sdbRelease(pMnode->pSdb, pObj); mError("bnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); return -1; } - sdbRelease(pMnode->pSdb, pMnode); + sdbRelease(pMnode->pSdb, pObj); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index ea4cfa41c8..5ff3a79040 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -26,7 +26,7 @@ static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj); static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); -static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode); +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg); static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg); static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg); @@ -155,9 +155,9 @@ static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { return 0; } -static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode) { - mTrace("qnode:%d, perform update action, old_row:%p new_row:%p", pOldQnode->id, pOldQnode, pNewQnode); - pOldQnode->updateTime = pNewQnode->updateTime; +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew) { + mTrace("qnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + pOld->updateTime = pNew->updateTime; return 0; } @@ -251,6 +251,7 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { SQnodeObj *pObj = mndAcquireQnode(pMnode, pCreate->dnodeId); if (pObj != NULL) { mError("qnode:%d, qnode already exist", pObj->id); + terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST; mndReleaseQnode(pMnode, pObj); return -1; } @@ -370,11 +371,12 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { int32_t code = mndDropQnode(pMnode, pMsg, pObj); if (code != 0) { + sdbRelease(pMnode->pSdb, pObj); mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); return -1; } - sdbRelease(pMnode->pSdb, pMnode); + sdbRelease(pMnode->pSdb, pObj); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 3ab1ad4eaf..5d13211380 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -26,7 +26,7 @@ static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj); static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); -static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode); +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg); static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg); static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg); @@ -155,9 +155,9 @@ static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj) { return 0; } -static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode) { - mTrace("snode:%d, perform update action, old_row:%p new_row:%p", pOldSnode->id, pOldSnode, pNewSnode); - pOldSnode->updateTime = pNewSnode->updateTime; +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew) { + mTrace("snode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + pOld->updateTime = pNew->updateTime; return 0; } @@ -251,6 +251,7 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { SSnodeObj *pObj = mndAcquireSnode(pMnode, pCreate->dnodeId); if (pObj != NULL) { mError("snode:%d, snode already exist", pObj->id); + terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; mndReleaseSnode(pMnode, pObj); return -1; } @@ -370,11 +371,12 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { int32_t code = mndDropSnode(pMnode, pMsg, pObj); if (code != 0) { + sdbRelease(pMnode->pSdb, pObj); mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); return -1; } - sdbRelease(pMnode->pSdb, pMnode); + sdbRelease(pMnode->pSdb, pObj); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } From 0d4490bce89344a08db60843fa7e8fdced78e17e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 4 Jan 2022 22:32:38 -0500 Subject: [PATCH 13/54] TD-12696 bug fix for insert --- include/common/tmsg.h | 4 ---- source/libs/parser/src/dataBlockMgt.c | 2 +- source/libs/parser/src/insertParser.c | 10 ++++++---- source/libs/parser/test/insertParserTest.cpp | 8 ++------ source/libs/parser/test/mockCatalogService.cpp | 4 ++-- source/libs/planner/test/phyPlanTests.cpp | 2 +- 6 files changed, 12 insertions(+), 18 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index affab8903c..1b2b6a677a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -156,10 +156,6 @@ typedef struct { uint16_t port; } SEpAddr; -typedef struct { - int32_t numOfVnodes; -} SMsgDesc; - typedef struct SMsgHead { int32_t contLen; int32_t vgId; diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index b315ea613f..bc4eae7ae9 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -467,7 +467,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { - const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); + const int INSERT_HEAD_SIZE = sizeof(SSubmitMsg); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index c63e854051..73b1b17353 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -121,11 +121,13 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS } static void buildMsgHeader(SVgDataBlocks* blocks) { - SMsgDesc* desc = (SMsgDesc*)blocks->pData; - desc->numOfVnodes = htonl(1); - SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + // SMsgDesc* desc = (SMsgDesc*)blocks->pData; + // desc->numOfVnodes = htonl(1); + // SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + SSubmitMsg* submit = (SSubmitMsg*)blocks->pData; submit->header.vgId = htonl(blocks->vg.vgId); - submit->header.contLen = htonl(blocks->size - sizeof(SMsgDesc)); + // submit->header.contLen = htonl(blocks->size - sizeof(SMsgDesc)); + submit->header.contLen = htonl(blocks->size); submit->length = submit->header.contLen; submit->numOfBlocks = htonl(blocks->numOfTables); SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp index 5c175cd023..9b007fb36d 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/insertParserTest.cpp @@ -70,9 +70,7 @@ protected: for (size_t i = 0; i < num; ++i) { SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; - SMsgDesc* desc = (SMsgDesc*)(vg->pData); - cout << "numOfVnodes:" << ntohl(desc->numOfVnodes) << endl; - SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + SSubmitMsg* submit = (SSubmitMsg*)vg->pData; cout << "length:" << ntohl(submit->length) << ", numOfBlocks:" << ntohl(submit->numOfBlocks) << endl; int32_t numOfBlocks = ntohl(submit->numOfBlocks); SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); @@ -95,9 +93,7 @@ protected: SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); ASSERT_EQ(vg->numOfTables, numOfTables); ASSERT_GE(vg->size, 0); - SMsgDesc* desc = (SMsgDesc*)(vg->pData); - ASSERT_EQ(ntohl(desc->numOfVnodes), 1); - SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + SSubmitMsg* submit = (SSubmitMsg*)vg->pData; ASSERT_GE(ntohl(submit->length), 0); ASSERT_GE(ntohl(submit->numOfBlocks), 0); int32_t numOfBlocks = ntohl(submit->numOfBlocks); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 3be358fec8..bdd023be83 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -27,7 +27,7 @@ std::unique_ptr mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { - assert(colId_ < schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); + assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); SSchema* col = schema()->schema + colId_; col->type = type; col->colId = colId_++; @@ -66,7 +66,7 @@ private: return std::unique_ptr(new TableBuilder(meta)); } - TableBuilder(STableMeta* schemaMeta) : colId_(0), rowsize_(0), meta_(new MockTableMeta()) { + TableBuilder(STableMeta* schemaMeta) : colId_(1), rowsize_(0), meta_(new MockTableMeta()) { meta_->schema.reset(schemaMeta); } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index b6f9612653..3f6f5ab77b 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -33,7 +33,7 @@ protected: void pushScan(const string& db, const string& table, int32_t scanOp) { shared_ptr meta = mockCatalogService->getTableMeta(db, table); EXPECT_TRUE(meta); - unique_ptr scan((SQueryPlanNode*)calloc(1, sizeof(SQueryPlanNode))); + unique_ptr scan((SQueryPlanNode*)myCalloc(1, sizeof(SQueryPlanNode))); scan->info.type = scanOp; scan->numOfCols = meta->schema->tableInfo.numOfColumns; scan->pSchema = (SSchema*)myCalloc(1, sizeof(SSchema) * scan->numOfCols); From 17b9917bfab758917811e9f8d164d4341a6452e4 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 4 Jan 2022 22:43:29 -0500 Subject: [PATCH 14/54] TD-12696 bug fix for insert --- source/libs/parser/src/insertParser.c | 4 ---- source/libs/parser/test/mockCatalogService.cpp | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 73b1b17353..8b3c328cce 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -121,12 +121,8 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS } static void buildMsgHeader(SVgDataBlocks* blocks) { - // SMsgDesc* desc = (SMsgDesc*)blocks->pData; - // desc->numOfVnodes = htonl(1); - // SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); SSubmitMsg* submit = (SSubmitMsg*)blocks->pData; submit->header.vgId = htonl(blocks->vg.vgId); - // submit->header.contLen = htonl(blocks->size - sizeof(SMsgDesc)); submit->header.contLen = htonl(blocks->size); submit->length = submit->header.contLen; submit->numOfBlocks = htonl(blocks->numOfTables); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index bdd023be83..2a0f6b38eb 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -28,7 +28,7 @@ class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); - SSchema* col = schema()->schema + colId_; + SSchema* col = schema()->schema + (colId_ - 1); col->type = type; col->colId = colId_++; col->bytes = bytes; From 9dd5922d150bb6f5c2232ea638201d8d646a37c9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 4 Jan 2022 22:36:24 -0800 Subject: [PATCH 15/54] error returned when trans rollback --- source/dnode/mnode/impl/inc/mndDef.h | 8 ++++---- source/dnode/mnode/impl/src/mndTrans.c | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 267a3f6cf5..93a5722842 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -64,10 +64,10 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_UNDO_LOG = 3, - TRN_STAGE_UNDO_ACTION = 4, - TRN_STAGE_COMMIT_LOG = 5, - TRN_STAGE_COMMIT = 6, + TRN_STAGE_COMMIT_LOG = 3, + TRN_STAGE_COMMIT = 4, + TRN_STAGE_UNDO_ACTION = 5, + TRN_STAGE_UNDO_LOG = 6, TRN_STAGE_ROLLBACK = 7, TRN_STAGE_FINISHED = 8 } ETrnStage; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index ee4a49ffdc..82fe296166 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -496,10 +496,15 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } static void mndTransSendRpcRsp(STrans *pTrans) { - if (pTrans->rpcHandle != NULL) { - mDebug("trans:%d, send rsp, ahandle:%p code:0x%x", pTrans->id, pTrans->rpcAHandle, pTrans->code & 0xFFFF); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle}; - rpcSendResponse(&rspMsg); + if (pTrans->stage == TRN_STAGE_FINISHED || pTrans->stage == TRN_STAGE_UNDO_LOG || + pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { + if (pTrans->rpcHandle != NULL) { + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + pTrans->rpcAHandle); + SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle}; + rpcSendResponse(&rspMsg); + pTrans->rpcHandle = NULL; + } } } @@ -764,7 +769,6 @@ static bool mndTransPerformCommitLogStage(SMnode *pMnode, STrans *pTrans) { pTrans->failedTimes++; mError("trans:%d, stage keep on commitLog since %s", pTrans->id, terrstr()); continueExec = false; - ; } return continueExec; @@ -791,7 +795,7 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); if (code == 0) { - pTrans->stage = TRN_STAGE_REDO_LOG; + pTrans->stage = TRN_STAGE_UNDO_LOG; mDebug("trans:%d, stage from undoAction to undoLog", pTrans->id); continueExec = true; } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { @@ -814,7 +818,6 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { pTrans->stage = TRN_STAGE_FINISHED; mDebug("trans:%d, stage from rollback to finished", pTrans->id); continueExec = true; - ; } else { pTrans->failedTimes++; mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); @@ -880,9 +883,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { } } - if (pTrans->stage == TRN_STAGE_FINISHED) { - mndTransSendRpcRsp(pTrans); - } + mndTransSendRpcRsp(pTrans); } static int32_t mndProcessTransMsg(SMnodeMsg *pMsg) { From 4728b159c796b36a298c5f0ce9bb72debcccf500 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 5 Jan 2022 16:00:06 +0800 Subject: [PATCH 16/54] feature/qnode --- include/libs/catalog/catalog.h | 19 ++- source/libs/catalog/inc/catalogInt.h | 8 ++ source/libs/catalog/src/catalog.c | 166 +++++++++++++++++----- source/libs/catalog/test/catalogTests.cpp | 7 +- 4 files changed, 154 insertions(+), 46 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 3916898829..e39236ea76 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -50,6 +50,7 @@ typedef struct SCatalogCfg { uint32_t maxDBCacheNum; } SCatalogCfg; + int32_t catalogInit(SCatalogCfg *cfg); /** @@ -87,15 +88,28 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB */ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +/** + * Get a super table's meta data. + * @param pCatalog (input, got with catalogGetHandle) + * @param pTransporter (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pTableName (input, table name, NOT including db name) + * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @return error code + */ +int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); + + /** * Force renew a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) * @param pTableName (input, table name, NOT including db name) + * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); /** * Force renew a table's local cached meta data and get the new one. @@ -104,9 +118,10 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, co * @param pMgmtEps (input, mnode EPs) * @param pTableName (input, table name, NOT including db name) * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); /** diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f426139c14..c4f2b54cf8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -64,6 +64,14 @@ typedef struct SCatalogMgmt { typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); +#define CTG_IS_STABLE(isSTable) (1 == (isSTable)) +#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable)) +#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0) +#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) +#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) + +#define CTG_TABLE_NOT_EXIST(code) (code == TSDB_CODE_TDB_INVALID_TABLE_ID) + #define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index abcfafa786..f7139f21b8 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -105,6 +105,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN } *exist = 1; + + tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_SUCCESS; @@ -143,6 +145,29 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN return TSDB_CODE_SUCCESS; } +int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { + if (NULL == pCatalog->tableCache.cache) { + return TSDB_CODE_SUCCESS; + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); + + size_t sz = 0; + STableMeta *pTableMeta = NULL; + + taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)&pTableMeta, &sz); + + if (NULL == pTableMeta) { + return TSDB_CODE_SUCCESS; + } + + *tbType = pTableMeta->tableType; + + return TSDB_CODE_SUCCESS; +} + + void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { epSet->inUse = 0; epSet->numOfEps = vgroupInfo->numOfEps; @@ -153,14 +178,7 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { } } -int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == output) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - +int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -179,6 +197,12 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { + output->metaNum = 0; + ctgDebug("tbmeta:%s not exist in mnode", tbFullName); + return TSDB_CODE_SUCCESS; + } + ctgError("error rsp for table meta, code:%x", rpcRsp.code); CTG_ERR_RET(rpcRsp.code); } @@ -188,6 +212,13 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE return TSDB_CODE_SUCCESS; } +int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); + + return ctgGetTableMetaFromMnodeImpl(pCatalog, pRpc, pMgmtEps, tbFullName, output); +} + int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { @@ -197,7 +228,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE char dbFullName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFullName); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = pTableName->tname}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)pTableName->tname}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; @@ -218,6 +249,12 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { + output->metaNum = 0; + ctgDebug("tbmeta:%s not exist in vnode", pTableName->tname); + return TSDB_CODE_SUCCESS; + } + ctgError("error rsp for table meta, code:%x", rpcRsp.code); CTG_ERR_RET(rpcRsp.code); } @@ -322,22 +359,28 @@ _return: CTG_RET(TSDB_CODE_SUCCESS); } -int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta) { +int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; - if (!forceUpdate) { + if (!forceUpdate) { CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); - if (exist) { + if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } + } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { + int32_t tbType = 0; + + CTG_ERR_RET(ctgGetTableTypeFromCache(pCatalog, pTableName, &tbType)); + + CTG_SET_STABLE(isSTable, tbType); } - CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pTableName)); + CTG_ERR_RET(ctgRenewTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, isSTable)); CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); @@ -364,19 +407,27 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out } if (NULL == pCatalog->tableCache.cache) { - pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == pCatalog->tableCache.cache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.cache, NULL, cache)) { + taosHashCleanup(cache); + } } if (NULL == pCatalog->tableCache.stableCache) { - pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == pCatalog->tableCache.stableCache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == cache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.stableCache, NULL, cache)) { + taosHashCleanup(cache); + } } if (output->metaNum == 2) { @@ -481,6 +532,50 @@ int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, SD return TSDB_CODE_SUCCESS; } +int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { + if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + SVgroupInfo vgroupInfo = {0}; + int32_t code = 0; + + CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + + STableMetaOutput voutput = {0}; + STableMetaOutput moutput = {0}; + STableMetaOutput *output = &voutput; + + if (CTG_IS_STABLE(isSTable)) { + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); + + if (0 == moutput.metaNum) { + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); + } else { + output = &moutput; + } + } else { + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); + + if (voutput.metaNum > 0 && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + + tfree(voutput.tbMeta); + voutput.tbMeta = moutput.tbMeta; + moutput.tbMeta = NULL; + } + } + + CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, output)); + +_return: + + tfree(voutput.tbMeta); + tfree(moutput.tbMeta); + + CTG_RET(code); +} + int32_t catalogInit(SCatalogCfg *cfg) { if (ctgMgmt.pCluster) { @@ -643,11 +738,15 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB } if (NULL == pCatalog->dbCache.cache) { - pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == pCatalog->dbCache.cache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == cache) { ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } + + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache.cache, NULL, cache)) { + taosHashCleanup(cache); + } } else { CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); } @@ -672,34 +771,23 @@ _return: } int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { - return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta); + return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1); } -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName) { +int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { + return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1); +} + +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - SVgroupInfo vgroupInfo = {0}; - int32_t code = 0; - - CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); - - STableMetaOutput output = {0}; - - CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &output)); - - //CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output)); - - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); - -_return: - tfree(output.tbMeta); - CTG_RET(code); + return ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable); } -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { - return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { + return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable); } int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1d8a48dfcb..4fc53e5f18 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -45,7 +45,7 @@ void ctgTestSetPrepareSTableMeta(); bool ctgTestStop = false; bool ctgTestEnableSleep = false; -bool ctgTestDeadLoop = true; +bool ctgTestDeadLoop = false; int32_t ctgTestCurrentVgVersion = 0; int32_t ctgTestVgVersion = 1; @@ -600,7 +600,6 @@ void *ctgTestSetCtableMetaThread(void *param) { } -#if 0 TEST(tableMeta, normalTable) { struct SCatalog* pCtg = NULL; @@ -768,7 +767,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); tableMeta = NULL; - code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); + code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); @@ -999,8 +998,6 @@ TEST(multiThread, getSetDbVgroupCase) { catalogDestroy(); } -#endif - TEST(multiThread, ctableMeta) { struct SCatalog* pCtg = NULL; From 3bd42107652d23ae7a110442ef4595db30ea979e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 01:31:13 -0800 Subject: [PATCH 17/54] When an error occurs in rollback or retry, the failed result is returned first --- include/dnode/mnode/sdb/sdb.h | 1 + source/dnode/mnode/impl/inc/mndTrans.h | 1 + source/dnode/mnode/impl/src/mndBnode.c | 111 ++++++------ source/dnode/mnode/impl/src/mndQnode.c | 111 ++++++------ source/dnode/mnode/impl/src/mndSnode.c | 109 ++++++------ source/dnode/mnode/impl/src/mndTrans.c | 43 +++-- source/dnode/mnode/impl/test/bnode/bnode.cpp | 168 +++++++++++++++++-- source/dnode/mnode/impl/test/qnode/qnode.cpp | 146 +++++++++++++++- source/dnode/mnode/impl/test/snode/snode.cpp | 168 +++++++++++++++++-- source/dnode/mnode/sdb/src/sdbHash.c | 4 + 10 files changed, 655 insertions(+), 207 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 497da71c13..c7198eee6f 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -94,6 +94,7 @@ typedef struct SSdbRaw SSdbRaw; typedef struct SSdbRow SSdbRow; typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyType; typedef enum { + SDB_STATUS_INIT = 0, SDB_STATUS_CREATING = 1, SDB_STATUS_UPDATING = 2, SDB_STATUS_DROPPING = 3, diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index bd053d91b6..fda3fed13d 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -28,6 +28,7 @@ typedef struct { int8_t msgSent; int8_t msgReceived; int32_t errCode; + int32_t acceptableCode; int32_t contLen; void *pCont; } STransAction; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 312438a535..14d49213e9 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -59,9 +59,8 @@ int32_t mndInitBnode(SMnode *pMnode) { void mndCleanupBnode(SMnode *pMnode) {} static SBnodeObj *mndAcquireBnode(SMnode *pMnode, int32_t bnodeId) { - SSdb *pSdb = pMnode->pSdb; - SBnodeObj *pObj = sdbAcquire(pSdb, SDB_BNODE, &bnodeId); - if (pObj == NULL) { + SBnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_BNODE, &bnodeId); + if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; } return pObj; @@ -169,6 +168,14 @@ static int32_t mndSetCreateBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { return 0; } +static int32_t mndSetCreateBnodeUndoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pUndoRaw = mndBnodeActionEncode(pObj); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); if (pCommitRaw == NULL) return -1; @@ -190,6 +197,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pMsg; action.contLen = sizeof(SDCreateBnodeReq); action.msgType = TDMT_DND_CREATE_BNODE; + action.acceptableCode = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -199,39 +207,47 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } +static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDDropBnodeReq *pMsg = malloc(sizeof(SDDropBnodeReq)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropBnodeReq); + action.msgType = TDMT_DND_DROP_BNODE; + action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; + + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { + int32_t code = -1; + SBnodeObj bnodeObj = {0}; bnodeObj.id = pDnode->id; bnodeObj.createdTime = taosGetTimestampMs(); bnodeObj.updateTime = bnodeObj.createdTime; - int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("bnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - goto CREATE_BNODE_OVER; - } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + if (pTrans == NULL) goto CREATE_BNODE_OVER; + mDebug("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); - - if (mndSetCreateBnodeRedoLogs(pTrans, &bnodeObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto CREATE_BNODE_OVER; - } - - if (mndSetCreateBnodeCommitLogs(pTrans, &bnodeObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto CREATE_BNODE_OVER; - } - - if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &bnodeObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_BNODE_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto CREATE_BNODE_OVER; - } + if (mndSetCreateBnodeRedoLogs(pTrans, &bnodeObj) != 0) goto CREATE_BNODE_OVER; + if (mndSetCreateBnodeUndoLogs(pTrans, &bnodeObj) != 0) goto CREATE_BNODE_OVER; + if (mndSetCreateBnodeCommitLogs(pTrans, &bnodeObj) != 0) goto CREATE_BNODE_OVER; + if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &bnodeObj) != 0) goto CREATE_BNODE_OVER; + if (mndSetCreateBnodeUndoActions(pTrans, pDnode, &bnodeObj) != 0) goto CREATE_BNODE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_BNODE_OVER; code = 0; @@ -254,6 +270,9 @@ static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_MND_BNODE_ALREADY_EXIST; mndReleaseBnode(pMnode, pObj); return -1; + } else if (terrno != TSDB_CODE_MND_BNODE_NOT_EXIST) { + mError("bnode:%d, failed to create bnode since %s", pCreate->dnodeId, terrstr()); + return -1; } SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); @@ -303,6 +322,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn action.pCont = pMsg; action.contLen = sizeof(SDDropBnodeReq); action.msgType = TDMT_DND_DROP_BNODE; + action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -314,33 +334,15 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pMsg, SBnodeObj *pObj) { int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("bnode:%d, failed to drop since %s", pObj->id, terrstr()); - goto DROP_BNODE_OVER; - } + if (pTrans == NULL) goto DROP_BNODE_OVER; mDebug("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); - - if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto DROP_BNODE_OVER; - } - - if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto DROP_BNODE_OVER; - } - - if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_BNODE_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto DROP_BNODE_OVER; - } + if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) goto DROP_BNODE_OVER; + if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) goto DROP_BNODE_OVER; + if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto DROP_BNODE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_BNODE_OVER; code = 0; @@ -364,8 +366,7 @@ static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { SBnodeObj *pObj = mndAcquireBnode(pMnode, pDrop->dnodeId); if (pObj == NULL) { - mError("bnode:%d, not exist", pDrop->dnodeId); - terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; + mError("bnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 5ff3a79040..6951aa8717 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -59,9 +59,8 @@ int32_t mndInitQnode(SMnode *pMnode) { void mndCleanupQnode(SMnode *pMnode) {} static SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { - SSdb *pSdb = pMnode->pSdb; - SQnodeObj *pObj = sdbAcquire(pSdb, SDB_QNODE, &qnodeId); - if (pObj == NULL) { + SQnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_QNODE, &qnodeId); + if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_QNODE_NOT_EXIST; } return pObj; @@ -169,6 +168,14 @@ static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { return 0; } +static int32_t mndSetCreateQnodeUndoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pUndoRaw = mndQnodeActionEncode(pObj); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + static int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); if (pCommitRaw == NULL) return -1; @@ -190,6 +197,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pMsg; action.contLen = sizeof(SDCreateQnodeReq); action.msgType = TDMT_DND_CREATE_QNODE; + action.acceptableCode = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -199,39 +207,47 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } +static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDDropQnodeReq *pMsg = malloc(sizeof(SDDropQnodeReq)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropQnodeReq); + action.msgType = TDMT_DND_DROP_QNODE; + action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; + + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { + int32_t code = -1; + SQnodeObj qnodeObj = {0}; qnodeObj.id = pDnode->id; qnodeObj.createdTime = taosGetTimestampMs(); qnodeObj.updateTime = qnodeObj.createdTime; - int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - goto CREATE_QNODE_OVER; - } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + if (pTrans == NULL) goto CREATE_QNODE_OVER; + mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); - - if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto CREATE_QNODE_OVER; - } - - if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto CREATE_QNODE_OVER; - } - - if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_QNODE_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto CREATE_QNODE_OVER; - } + if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto CREATE_QNODE_OVER; + if (mndSetCreateQnodeUndoLogs(pTrans, &qnodeObj) != 0) goto CREATE_QNODE_OVER; + if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) goto CREATE_QNODE_OVER; + if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) goto CREATE_QNODE_OVER; + if (mndSetCreateQnodeUndoActions(pTrans, pDnode, &qnodeObj) != 0) goto CREATE_QNODE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_QNODE_OVER; code = 0; @@ -254,6 +270,9 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_MND_QNODE_ALREADY_EXIST; mndReleaseQnode(pMnode, pObj); return -1; + } else if (terrno != TSDB_CODE_MND_QNODE_NOT_EXIST) { + mError("qnode:%d, failed to create qnode since %s", pCreate->dnodeId, terrstr()); + return -1; } SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); @@ -303,6 +322,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.pCont = pMsg; action.contLen = sizeof(SDDropQnodeReq); action.msgType = TDMT_DND_DROP_QNODE; + action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -314,33 +334,15 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pMsg, SQnodeObj *pObj) { int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("qnode:%d, failed to drop since %s", pObj->id, terrstr()); - goto DROP_QNODE_OVER; - } + if (pTrans == NULL) goto DROP_QNODE_OVER; mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); - - if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto DROP_QNODE_OVER; - } - - if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto DROP_QNODE_OVER; - } - - if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_QNODE_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto DROP_QNODE_OVER; - } + if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) goto DROP_QNODE_OVER; + if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) goto DROP_QNODE_OVER; + if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto DROP_QNODE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_QNODE_OVER; code = 0; @@ -364,8 +366,7 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { SQnodeObj *pObj = mndAcquireQnode(pMnode, pDrop->dnodeId); if (pObj == NULL) { - mError("qnode:%d, not exist", pDrop->dnodeId); - terrno = TSDB_CODE_MND_QNODE_NOT_EXIST; + mError("qnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 5d13211380..7221a2fbf4 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -59,9 +59,8 @@ int32_t mndInitSnode(SMnode *pMnode) { void mndCleanupSnode(SMnode *pMnode) {} static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { - SSdb *pSdb = pMnode->pSdb; - SSnodeObj *pObj = sdbAcquire(pSdb, SDB_SNODE, &snodeId); - if (pObj == NULL) { + SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId); + if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_SNODE_NOT_EXIST; } return pObj; @@ -169,6 +168,14 @@ static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { return 0; } +static int32_t mndSetCreateSnodeUndoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pUndoRaw = mndSnodeActionEncode(pObj); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); if (pCommitRaw == NULL) return -1; @@ -190,6 +197,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pMsg; action.contLen = sizeof(SDCreateSnodeReq); action.msgType = TDMT_DND_CREATE_SNODE; + action.acceptableCode = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -199,39 +207,48 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S return 0; } +static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDDropSnodeReq *pMsg = malloc(sizeof(SDDropSnodeReq)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropSnodeReq); + action.msgType = TDMT_DND_DROP_SNODE; + action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; + + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { + int32_t code = -1; + SSnodeObj snodeObj = {0}; snodeObj.id = pDnode->id; snodeObj.createdTime = taosGetTimestampMs(); snodeObj.updateTime = snodeObj.createdTime; - int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); - goto CREATE_SNODE_OVER; - } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + if (pTrans == NULL) goto CREATE_SNODE_OVER; + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); - if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto CREATE_SNODE_OVER; - } - - if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto CREATE_SNODE_OVER; - } - - if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_SNODE_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto CREATE_SNODE_OVER; - } + if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) goto CREATE_SNODE_OVER; + if (mndSetCreateSnodeUndoLogs(pTrans, &snodeObj) != 0) goto CREATE_SNODE_OVER; + if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) goto CREATE_SNODE_OVER; + if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) goto CREATE_SNODE_OVER; + if (mndSetCreateSnodeUndoActions(pTrans, pDnode, &snodeObj) != 0) goto CREATE_SNODE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_SNODE_OVER; code = 0; @@ -254,6 +271,9 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST; mndReleaseSnode(pMnode, pObj); return -1; + } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) { + mError("snode:%d, failed to create snode since %s", pCreate->dnodeId, terrstr()); + return -1; } SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); @@ -303,6 +323,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.pCont = pMsg; action.contLen = sizeof(SDDropSnodeReq); action.msgType = TDMT_DND_DROP_SNODE; + action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -314,33 +335,16 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pMsg, SSnodeObj *pObj) { int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); - goto DROP_SNODE_OVER; - } + if (pTrans == NULL) goto DROP_SNODE_OVER; mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); - if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto DROP_SNODE_OVER; - } - - if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto DROP_SNODE_OVER; - } - - if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_SNODE_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto DROP_SNODE_OVER; - } + if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) goto DROP_SNODE_OVER; + if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) goto DROP_SNODE_OVER; + if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto DROP_SNODE_OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_SNODE_OVER; code = 0; @@ -364,8 +368,7 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { SSnodeObj *pObj = mndAcquireSnode(pMnode, pDrop->dnodeId); if (pObj == NULL) { - mError("snode:%d, not exist", pDrop->dnodeId); - terrno = TSDB_CODE_MND_SNODE_NOT_EXIST; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 82fe296166..c1686c2923 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -143,6 +143,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { STransAction *pAction = taosArrayGet(pTrans->redoActions, i); SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) } @@ -151,6 +152,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { STransAction *pAction = taosArrayGet(pTrans->undoActions, i); SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), TRANS_ENCODE_OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, TRANS_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) } @@ -253,6 +255,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < redoActionNum; ++i) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) action.pCont = malloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; @@ -264,6 +267,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < undoActionNum; ++i) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), TRANS_DECODE_OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &action.contLen, TRANS_DECODE_OVER) action.pCont = malloc(action.contLen); if (action.pCont == NULL) goto TRANS_DECODE_OVER; @@ -496,16 +500,32 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } static void mndTransSendRpcRsp(STrans *pTrans) { - if (pTrans->stage == TRN_STAGE_FINISHED || pTrans->stage == TRN_STAGE_UNDO_LOG || - pTrans->stage == TRN_STAGE_UNDO_ACTION || pTrans->stage == TRN_STAGE_ROLLBACK) { - if (pTrans->rpcHandle != NULL) { - mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, - pTrans->rpcAHandle); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle}; - rpcSendResponse(&rspMsg); - pTrans->rpcHandle = NULL; + bool sendRsp = false; + + if (pTrans->stage == TRN_STAGE_FINISHED) { + sendRsp = true; + } + + if (pTrans->policy == TRN_POLICY_ROLLBACK) { + if (pTrans->stage == TRN_STAGE_UNDO_LOG || pTrans->stage == TRN_STAGE_UNDO_ACTION || + pTrans->stage == TRN_STAGE_ROLLBACK) { + sendRsp = true; } } + + if (pTrans->policy == TRN_POLICY_RETRY) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 0) { + sendRsp = true; + } + } + + if (sendRsp && pTrans->rpcHandle != NULL) { + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, + pTrans->rpcAHandle); + SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle}; + rpcSendResponse(&rspMsg); + pTrans->rpcHandle = NULL; + } } void mndTransProcessRsp(SMnodeMsg *pMsg) { @@ -547,7 +567,8 @@ void mndTransProcessRsp(SMnodeMsg *pMsg) { pAction->errCode = pMsg->rpcMsg.code; } - mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->rpcMsg.code); + mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pMsg->rpcMsg.code, + pAction->acceptableCode); mndTransExecute(pMnode, pTrans); HANDLE_ACTION_RSP_OVER: @@ -647,7 +668,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA if (pAction == NULL) continue; if (pAction->msgSent && pAction->msgReceived) { numOfReceived++; - if (pAction->errCode != 0) { + if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { errCode = pAction->errCode; } } @@ -695,7 +716,7 @@ static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans) { } else { pTrans->code = terrno; pTrans->stage = TRN_STAGE_UNDO_LOG; - mError("trans:%d, stage from redoLog to undoLog", pTrans->id); + mError("trans:%d, stage from redoLog to undoLog since %s", pTrans->id, terrstr()); } return continueExec; diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index fa06d9ac69..07dc163df5 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -50,7 +50,18 @@ TEST_F(MndTestBnode, 01_Show_Bnode) { EXPECT_EQ(test.GetShowRows(), 0); } -TEST_F(MndTestBnode, 02_Create_Bnode_Invalid_Id) { +TEST_F(MndTestBnode, 02_Create_Bnode) { + { + int32_t contLen = sizeof(SMCreateBnodeReq); + + SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } + { int32_t contLen = sizeof(SMCreateBnodeReq); @@ -63,11 +74,6 @@ TEST_F(MndTestBnode, 02_Create_Bnode_Invalid_Id) { test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); CHECK_META("show bnodes", 3); - - CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); - CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); - CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); EXPECT_EQ(test.GetShowRows(), 1); @@ -75,24 +81,21 @@ TEST_F(MndTestBnode, 02_Create_Bnode_Invalid_Id) { CheckBinary("localhost:9018", TSDB_EP_LEN); CheckTimestamp(); } -} -TEST_F(MndTestBnode, 03_Create_Bnode_Invalid_Id) { { int32_t contLen = sizeof(SMCreateBnodeReq); SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + pReq->dnodeId = htonl(1); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_BNODE_ALREADY_EXIST); } } -TEST_F(MndTestBnode, 04_Create_Bnode) { +TEST_F(MndTestBnode, 03_Drop_Bnode) { { - // create dnode int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); @@ -110,7 +113,6 @@ TEST_F(MndTestBnode, 04_Create_Bnode) { } { - // create bnode int32_t contLen = sizeof(SMCreateBnodeReq); SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); @@ -133,7 +135,6 @@ TEST_F(MndTestBnode, 04_Create_Bnode) { } { - // drop bnode int32_t contLen = sizeof(SMDropBnodeReq); SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); @@ -151,4 +152,143 @@ TEST_F(MndTestBnode, 04_Create_Bnode) { CheckBinary("localhost:9018", TSDB_EP_LEN); CheckTimestamp(); } + + { + int32_t contLen = sizeof(SMDropBnodeReq); + + SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_BNODE_NOT_EXIST); + } +} + +TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { + { + // send message first, then dnode2 crash, result is returned, and rollback is started + int32_t contLen = sizeof(SMCreateBnodeReq); + + SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + { + // continue send message, bnode is creating + int32_t contLen = sizeof(SMCreateBnodeReq); + + SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + } + + { + // continue send message, bnode is creating + int32_t contLen = sizeof(SMDropBnodeReq); + + SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + } + + { + // server start, wait until the rollback finished + server2.DoStart(); + taosMsleep(1000); + + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateBnodeReq); + + SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + if (pMsg->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + } +} + +TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { + { + // send message first, then dnode2 crash, result is returned, and rollback is started + int32_t contLen = sizeof(SMDropBnodeReq); + + SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + { + // continue send message, bnode is dropping + int32_t contLen = sizeof(SMCreateBnodeReq); + + SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + // continue send message, bnode is dropping + int32_t contLen = sizeof(SMDropBnodeReq); + + SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + // server start, wait until the rollback finished + server2.DoStart(); + taosMsleep(1000); + + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateBnodeReq); + + SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + if (pMsg->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 3fdd5315a4..612b138ccc 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -90,13 +90,12 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_QNODE_ALREADY_EXIST); } } -TEST_F(MndTestQnode, 04_Create_Qnode) { +TEST_F(MndTestQnode, 03_Drop_Qnode) { { - // create dnode int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); @@ -114,7 +113,6 @@ TEST_F(MndTestQnode, 04_Create_Qnode) { } { - // create qnode int32_t contLen = sizeof(SMCreateQnodeReq); SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); @@ -137,7 +135,6 @@ TEST_F(MndTestQnode, 04_Create_Qnode) { } { - // drop qnode int32_t contLen = sizeof(SMDropQnodeReq); SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); @@ -155,4 +152,143 @@ TEST_F(MndTestQnode, 04_Create_Qnode) { CheckBinary("localhost:9014", TSDB_EP_LEN); CheckTimestamp(); } + + { + int32_t contLen = sizeof(SMDropQnodeReq); + + SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_QNODE_NOT_EXIST); + } +} + +TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { + { + // send message first, then dnode2 crash, result is returned, and rollback is started + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + { + // continue send message, qnode is creating + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + } + + { + // continue send message, qnode is creating + int32_t contLen = sizeof(SMDropQnodeReq); + + SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + } + + { + // server start, wait until the rollback finished + server2.DoStart(); + taosMsleep(1000); + + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + if (pMsg->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + } +} + +TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { + { + // send message first, then dnode2 crash, result is returned, and rollback is started + int32_t contLen = sizeof(SMDropQnodeReq); + + SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + { + // continue send message, qnode is dropping + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + // continue send message, qnode is dropping + int32_t contLen = sizeof(SMDropQnodeReq); + + SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + // server start, wait until the rollback finished + server2.DoStart(); + taosMsleep(1000); + + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + if (pMsg->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 4819bc556c..cebb5fe113 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -50,7 +50,18 @@ TEST_F(MndTestSnode, 01_Show_Snode) { EXPECT_EQ(test.GetShowRows(), 0); } -TEST_F(MndTestSnode, 02_Create_Snode_Invalid_Id) { +TEST_F(MndTestSnode, 02_Create_Snode) { + { + int32_t contLen = sizeof(SMCreateSnodeReq); + + SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } + { int32_t contLen = sizeof(SMCreateSnodeReq); @@ -63,11 +74,6 @@ TEST_F(MndTestSnode, 02_Create_Snode_Invalid_Id) { test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); CHECK_META("show snodes", 3); - - CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); - CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); - CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); EXPECT_EQ(test.GetShowRows(), 1); @@ -75,24 +81,21 @@ TEST_F(MndTestSnode, 02_Create_Snode_Invalid_Id) { CheckBinary("localhost:9016", TSDB_EP_LEN); CheckTimestamp(); } -} -TEST_F(MndTestSnode, 03_Create_Snode_Invalid_Id) { { int32_t contLen = sizeof(SMCreateSnodeReq); SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + pReq->dnodeId = htonl(1); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_SNODE_ALREADY_EXIST); } } -TEST_F(MndTestSnode, 04_Create_Snode) { +TEST_F(MndTestSnode, 03_Drop_Snode) { { - // create dnode int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); @@ -110,7 +113,6 @@ TEST_F(MndTestSnode, 04_Create_Snode) { } { - // create snode int32_t contLen = sizeof(SMCreateSnodeReq); SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); @@ -133,7 +135,6 @@ TEST_F(MndTestSnode, 04_Create_Snode) { } { - // drop snode int32_t contLen = sizeof(SMDropSnodeReq); SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); @@ -151,4 +152,143 @@ TEST_F(MndTestSnode, 04_Create_Snode) { CheckBinary("localhost:9016", TSDB_EP_LEN); CheckTimestamp(); } + + { + int32_t contLen = sizeof(SMDropSnodeReq); + + SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_SNODE_NOT_EXIST); + } +} + +TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { + { + // send message first, then dnode2 crash, result is returned, and rollback is started + int32_t contLen = sizeof(SMCreateSnodeReq); + + SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + { + // continue send message, snode is creating + int32_t contLen = sizeof(SMCreateSnodeReq); + + SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + } + + { + // continue send message, snode is creating + int32_t contLen = sizeof(SMDropSnodeReq); + + SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + } + + { + // server start, wait until the rollback finished + server2.DoStart(); + taosMsleep(1000); + + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateSnodeReq); + + SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + if (pMsg->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + } +} + +TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { + { + // send message first, then dnode2 crash, result is returned, and rollback is started + int32_t contLen = sizeof(SMDropSnodeReq); + + SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + { + // continue send message, snode is dropping + int32_t contLen = sizeof(SMCreateSnodeReq); + + SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + // continue send message, snode is dropping + int32_t contLen = sizeof(SMDropSnodeReq); + + SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + } + + { + // server start, wait until the rollback finished + server2.DoStart(); + taosMsleep(1000); + + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateSnodeReq); + + SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + if (pMsg->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + } } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 733075757f..13b2c7daa5 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -69,6 +69,8 @@ static const char *sdbStatusStr(ESdbStatus status) { return "ready"; case SDB_STATUS_DROPPED: return "dropped"; + case SDB_STATUS_INIT: + return "init"; default: return "undefine"; } @@ -261,6 +263,8 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { } void *sdbAcquire(SSdb *pSdb, ESdbType type, void *pKey) { + terrno = 0; + SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; From 69349c38d47f20e707329073f37eed60d581d31e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 02:08:56 -0800 Subject: [PATCH 18/54] fix gtest errors --- source/dnode/mgmt/impl/test/profile/profile.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 77122d1bb9..b705a94af6 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -17,15 +17,15 @@ class DndTestProfile : public ::testing::Test { static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; + static int32_t connId; public: void SetUp() override {} void TearDown() override {} - - int32_t connId; }; Testbase DndTestProfile::test; +int32_t DndTestProfile::connId; TEST_F(DndTestProfile, 01_ConnectMsg) { int32_t contLen = sizeof(SConnectMsg); From fdca2ec4bfc1ce0f868a405dbae337533e9fc33a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 02:18:47 -0800 Subject: [PATCH 19/54] minor changes --- source/dnode/mgmt/impl/test/CMakeLists.txt | 2 - .../mgmt/impl/test/cluster/CMakeLists.txt | 11 ----- .../dnode/mgmt/impl/test/cluster/cluster.cpp | 41 ------------------- .../impl/test/show/CMakeLists.txt | 0 .../{mgmt => mnode}/impl/test/show/show.cpp | 0 5 files changed, 54 deletions(-) delete mode 100644 source/dnode/mgmt/impl/test/cluster/CMakeLists.txt delete mode 100644 source/dnode/mgmt/impl/test/cluster/cluster.cpp rename source/dnode/{mgmt => mnode}/impl/test/show/CMakeLists.txt (100%) rename source/dnode/{mgmt => mnode}/impl/test/show/show.cpp (100%) diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index da043f8fe2..b13edf7d5a 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -6,13 +6,11 @@ add_subdirectory(snode) # add_subdirectory(auth) # add_subdirectory(balance) -add_subdirectory(cluster) add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) add_subdirectory(profile) -add_subdirectory(show) add_subdirectory(stb) # add_subdirectory(sync) # add_subdirectory(telem) diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt deleted file mode 100644 index a63f3106e6..0000000000 --- a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. CLUSTER_SRC) -add_executable(dnode_test_cluster ${CLUSTER_SRC}) -target_link_libraries( - dnode_test_cluster - PUBLIC sut -) - -add_test( - NAME dnode_test_cluster - COMMAND dnode_test_cluster -) diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp deleted file mode 100644 index 7d9bff7b23..0000000000 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/** - * @file cluster.cpp - * @author slguan (slguan@taosdata.com) - * @brief DNODE module cluster-msg tests - * @version 0.1 - * @date 2021-12-15 - * - * @copyright Copyright (c) 2021 - * - */ - -#include "sut.h" - -class DndTestCluster : public ::testing::Test { - protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_cluster", 9030); } - static void TearDownTestSuite() { test.Cleanup(); } - - static Testbase test; - - public: - void SetUp() override {} - void TearDown() override {} -}; - -Testbase DndTestCluster::test; - -TEST_F(DndTestCluster, 01_ShowCluster) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, ""); - CHECK_META( "show cluster", 3); - CHECK_SCHEMA(0, TSDB_DATA_TYPE_BIGINT, 8, "id"); - CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); - CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - - test.SendShowRetrieveMsg(); - EXPECT_EQ(test.GetShowRows(), 1); - - IgnoreInt64(); - IgnoreBinary(TSDB_CLUSTER_ID_LEN); - CheckTimestamp(); -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mnode/impl/test/show/CMakeLists.txt similarity index 100% rename from source/dnode/mgmt/impl/test/show/CMakeLists.txt rename to source/dnode/mnode/impl/test/show/CMakeLists.txt diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp similarity index 100% rename from source/dnode/mgmt/impl/test/show/show.cpp rename to source/dnode/mnode/impl/test/show/show.cpp From 9e3a6804a9d5ca9e1403f154d0b078803e5d1c61 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 02:20:59 -0800 Subject: [PATCH 20/54] add show test --- source/dnode/mnode/impl/test/CMakeLists.txt | 2 +- .../dnode/mnode/impl/test/show/CMakeLists.txt | 8 ++-- source/dnode/mnode/impl/test/show/show.cpp | 37 +++++++++++++------ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 075ff38bc6..6cdc43344d 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -6,4 +6,4 @@ add_subdirectory(trans) add_subdirectory(qnode) add_subdirectory(snode) add_subdirectory(bnode) - +add_subdirectory(show) diff --git a/source/dnode/mnode/impl/test/show/CMakeLists.txt b/source/dnode/mnode/impl/test/show/CMakeLists.txt index 9786f65605..cc0706ca50 100644 --- a/source/dnode/mnode/impl/test/show/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/show/CMakeLists.txt @@ -1,11 +1,11 @@ aux_source_directory(. SHOW_SRC) -add_executable(dnode_test_show ${SHOW_SRC}) +add_executable(mnode_test_show ${SHOW_SRC}) target_link_libraries( - dnode_test_show + mnode_test_show PUBLIC sut ) add_test( - NAME dnode_test_show - COMMAND dnode_test_show + NAME mnode_test_show + COMMAND mnode_test_show ) diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 8622672758..6d2c9150f2 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -1,19 +1,19 @@ /** * @file show.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module show-msg tests - * @version 0.1 - * @date 2021-12-15 + * @brief MNODE module show tests + * @version 1.0 + * @date 2022-01-06 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestShow : public ::testing::Test { +class MndTestShow : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_show", 9091); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9020); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -23,9 +23,9 @@ class DndTestShow : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestShow::test; +Testbase MndTestShow::test; -TEST_F(DndTestShow, 01_ShowMsg_InvalidMsgMax) { +TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { int32_t contLen = sizeof(SShowMsg); SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); @@ -37,7 +37,7 @@ TEST_F(DndTestShow, 01_ShowMsg_InvalidMsgMax) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } -TEST_F(DndTestShow, 02_ShowMsg_InvalidMsgStart) { +TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { int32_t contLen = sizeof(SShowMsg); SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); @@ -49,12 +49,12 @@ TEST_F(DndTestShow, 02_ShowMsg_InvalidMsgStart) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } -TEST_F(DndTestShow, 02_ShowMsg_Conn) { +TEST_F(MndTestShow, 03_ShowMsg_Conn) { int32_t contLen = sizeof(SConnectMsg); SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dnode_test_show"); + strcpy(pReq->app, "mnode_test_show"); strcpy(pReq->db, ""); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); @@ -85,3 +85,18 @@ TEST_F(DndTestShow, 02_ShowMsg_Conn) { EXPECT_EQ(pRetrieveRsp->compressed, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); } + +TEST_F(MndTestShow, 04_ShowMsg_Cluster) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, ""); + CHECK_META( "show cluster", 3); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BIGINT, 8, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + IgnoreInt64(); + IgnoreBinary(TSDB_CLUSTER_ID_LEN); + CheckTimestamp(); +} \ No newline at end of file From 11d6b02aa4fb0c2097c7b17b7986c5d3305d03ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 02:35:34 -0800 Subject: [PATCH 21/54] minor changes --- source/dnode/mnode/impl/src/mndAuth.c | 1 - source/dnode/mnode/impl/src/mndCluster.c | 1 - source/dnode/mnode/impl/src/mndShow.c | 22 +++++++++++----------- source/dnode/mnode/impl/src/mndUser.c | 1 - 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index bcefbfde21..a2fc7a57b3 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE -#include "os.h" #include "mndAuth.h" int32_t mndInitAuth(SMnode *pMnode) { return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index b29f8276fe..8cb98a148f 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "mndCluster.h" #include "mndShow.h" -#include "mndTrans.h" #define TSDB_CLUSTER_VER_NUMBE 1 #define TSDB_CLUSTER_RESERVE_SIZE 64 diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 125c250614..81e15497bf 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -22,8 +22,8 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); -static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); +static int32_t mndProcessShowReq(SMnodeMsg *pMsg); +static int32_t mndProcessRetrieveReq(SMnodeMsg *pMsg); static bool mndCheckRetrieveFinished(SShowObj *pShow); int32_t mndInitShow(SMnode *pMnode) { @@ -36,8 +36,8 @@ int32_t mndInitShow(SMnode *pMnode) { return -1; } - mndSetMsgHandle(pMnode, TDMT_MND_SHOW, mndProcessShowMsg); - mndSetMsgHandle(pMnode, TDMT_MND_SHOW_RETRIEVE, mndProcessRetrieveMsg); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW, mndProcessShowReq); + mndSetMsgHandle(pMnode, TDMT_MND_SHOW_RETRIEVE, mndProcessRetrieveReq); return 0; } @@ -115,7 +115,7 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { taosCacheRelease(pMgmt->cache, (void **)(&pShow), forceRemove); } -static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { +static int32_t mndProcessShowReq(SMnodeMsg *pMnodeMsg) { SMnode *pMnode = pMnodeMsg->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont; @@ -168,14 +168,14 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { } } -static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { - SMnode *pMnode = pMnodeMsg->pMnode; +static int32_t mndProcessRetrieveReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; int32_t rowsRead = 0; - SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont; + SRetrieveTableMsg *pRetrieve = pMsg->rpcMsg.pCont; int64_t showId = htobe64(pRetrieve->showId); SShowObj *pShow = mndAcquireShowObj(pMnode, showId); @@ -227,7 +227,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { // if free flag is set, client wants to clean the resources if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { - rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); + rowsRead = (*retrieveFp)(pMsg, pShow, pRsp->data, rowsToRead); } mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); @@ -235,8 +235,8 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pMnodeMsg->pCont = pRsp; - pMnodeMsg->contLen = size; + pMsg->pCont = pRsp; + pMsg->contLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 3ffb979ed1..0891cf43b3 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "mndUser.h" #include "mndShow.h" -#include "mndSync.h" #include "mndTrans.h" #include "tkey.h" From 11eb2719f8df8ae2ece30fdf2be93fa65cdc8a40 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 5 Jan 2022 19:17:14 +0800 Subject: [PATCH 22/54] refactor code --- source/libs/index/src/index.c | 63 +++++++--- source/libs/index/src/index_cache.c | 73 +++++++---- source/libs/index/src/index_tfile.c | 181 ++++++++++++++++------------ 3 files changed, 193 insertions(+), 124 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b5a65a9fda..9c66cfe72f 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -63,7 +63,9 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { // pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -73,7 +75,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); - if (sIdx->tindex == NULL) { goto END; } + if (sIdx->tindex == NULL) { + goto END; + } sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; @@ -84,7 +88,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #endif END: - if (sIdx != NULL) { indexClose(sIdx); } + if (sIdx != NULL) { + indexClose(sIdx); + } *index = NULL; return -1; @@ -100,7 +106,9 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { indexCacheUnRef(*pCache); } + if (*pCache) { + indexCacheUnRef(*pCache); + } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); @@ -158,7 +166,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { return ret; } + if (ret != 0) { + return ret; + } } #endif @@ -188,7 +198,9 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } + for (int i = 0; i < tsz; i++) { + taosArrayPush(result, &tResult[i]); + } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -245,7 +257,9 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { return NULL; } + if (p == NULL) { + return NULL; + } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -267,7 +281,9 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { return NULL; } + if (t == NULL) { + return NULL; + } t->suid = suid; t->operType = oper; @@ -340,7 +356,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { return; } + if (results == NULL) { + return; + } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -396,16 +414,22 @@ static void indexDestroyTempResult(SArray* result) { taosArrayDestroy(result); } int indexFlushCacheTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); - if (pReader == NULL) { indexWarn("empty tfile reader found"); } + if (pReader == NULL) { + indexWarn("empty tfile reader found"); + } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); - if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } + if (tfileIter == NULL) { + indexWarn("empty tfile reader iterator"); + } SArray* result = taosArrayInit(1024, sizeof(void*)); @@ -452,10 +476,6 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { while (tn == true) { IterateValue* tv = tfileIter->getValue(tfileIter); TFileValue* tfv = tfileValueCreate(tv->colVal); - if (tv->val == NULL) { - // HO - printf("NO...."); - } taosArrayAddAll(tfv->tableId, tv->val); indexMergeSameKey(result, tfv); tn = tfileIter->next(tfileIter); @@ -476,7 +496,9 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayDestroy(value->val); value->val = NULL; } else { - if (value->val != NULL) { taosArrayClear(value->val); } + if (value->val != NULL) { + taosArrayClear(value->val); + } } free(value->colVal); value->colVal = NULL; @@ -499,11 +521,12 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { tfileWriterClose(tw); TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); - if (reader == NULL) { goto END; } + if (reader == NULL) { + goto END; + } TFileHeader* header = &reader->header; - ICacheKey key = { - .suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; + ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; pthread_mutex_lock(&sIdx->mtx); IndexTFile* ifile = (IndexTFile*)sIdx->tindex; diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 517cb1640d..d87c601b00 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -21,10 +21,6 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -// ref index_cache.h:22 -//#define CACHE_KEY_LEN(p) \ -// (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + -// sizeof(p->operType)) static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -119,7 +115,9 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); @@ -132,7 +130,9 @@ void indexCacheDestroyImm(IndexCache* cache) { } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { return; } + if (pCache == NULL) { + return; + } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -142,7 +142,9 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { return NULL; } + if (iiter == NULL) { + return NULL; + } pthread_mutex_lock(&cache->mtx); @@ -160,7 +162,9 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { return; } + if (iter == NULL) { + return; + } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -198,13 +202,17 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { return -1; } + if (cache == NULL) { + return -1; + } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { return -1; } + if (cache == NULL) { + return -1; + } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -235,7 +243,9 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u } static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { - if (mem == NULL) { return 0; } + if (mem == NULL) { + return 0; + } char* key = getIndexKey(ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); @@ -261,7 +271,9 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA return 0; } int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { return 0; } + if (cache == NULL) { + return 0; + } IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; @@ -275,14 +287,12 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; - // indexCacheDebug(pCache); int ret = indexQueryMem(mem, &ct, qtype, result, s); if (ret == 0 && *s != kTypeDeletion) { // continue search in imm ret = indexQueryMem(imm, &ct, qtype, result, s); } - // cacheTermDestroy(ct); indexMemUnRef(mem); indexMemUnRef(imm); @@ -291,23 +301,33 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { return; } + if (cache == NULL) { + return; + } int ref = T_REF_DEC(cache); - if (ref == 0) { indexCacheDestroy(cache); } + if (ref == 0) { + indexCacheDestroy(cache); + } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { return; } + if (tbl == NULL) { + return; + } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { return; } + if (tbl == NULL) { + return; + } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -317,7 +337,9 @@ void indexMemUnRef(MemTable* tbl) { } static void cacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { return; } + if (ct == NULL) { + return; + } free(ct->colVal); free(ct); } @@ -332,7 +354,9 @@ static int32_t compareKey(const void* l, const void* r) { // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { return rt->version - lt->version; } + if (cmp == 0) { + return rt->version - lt->version; + } return cmp; } @@ -352,11 +376,10 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { return false; } - IterateValue* iv = &itera->val; - if (iv->colVal != NULL && iv->val != NULL) { - // indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + if (iter == NULL) { + return false; } + IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); bool next = tSkipListIterNext(iter); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 753dbe87b1..958d511630 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -55,15 +55,14 @@ static void tfileGenFileFullName(char* fullname, const char* path, uint64_t s TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { return NULL; } + if (tcache == NULL) { + return NULL; + } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; SArray* files = tfileGetFileList(path); - - uint64_t suid; - int32_t colId, version; for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); @@ -74,12 +73,13 @@ TFileCache* tfileCacheCreate(const char* path) { } TFileReader* reader = tfileReaderCreate(wc); - if (reader == NULL) { goto End; } + if (reader == NULL) { + goto End; + } TFileHeader* header = &reader->header; + ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName)}; - char buf[128] = {0}; - ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName)}; - + char buf[128] = {0}; int32_t sz = indexSerialCacheKey(&key, buf); assert(sz < sizeof(buf)); taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); @@ -93,7 +93,9 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { return; } + if (tcache == NULL) { + return; + } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -114,7 +116,9 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { int32_t sz = indexSerialCacheKey(key, buf); assert(sz < sizeof(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } tfileReaderRef(*reader); return *reader; @@ -137,7 +141,9 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } // T_REF_INC(reader); reader->ctx = ctx; @@ -157,7 +163,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); @@ -197,7 +205,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); // indexInfo("open write file name %s", fullname); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); - if (wcx == NULL) { return NULL; } + if (wcx == NULL) { + return NULL; + } TFileHeader tfh = {0}; tfh.suid = suid; @@ -212,24 +222,15 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); - // indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); - if (wc == NULL) { return NULL; } + indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); + if (wc == NULL) { + return NULL; + } TFileReader* reader = tfileReaderCreate(wc); return reader; } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { - // char pathBuf[128] = {0}; - // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); - // TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType}; - // memcpy(header.colName, ); - - // char buf[TFILE_HADER_PRE_SIZE]; - // int len = TFILE_HADER_PRE_SIZE; - // if (len != ctx->write(ctx, buf, len)) { - // indexError("index: %" PRIu64 " failed to write header info", header->suid); - // return NULL; - //} TFileWriter* tw = calloc(1, sizeof(TFileWriter)); if (tw == NULL) { indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); @@ -278,34 +279,14 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - // if (offset + ttsz >= bufLimit) { - // // batch write - // indexInfo("offset: %d, ttsz: %d", offset, ttsz); - // // std::cout << "offset: " << offset << std::endl; - // // std::cout << "ttsz:" << ttsz < < < std::endl; - // tw->ctx->write(tw->ctx, buf, offset); - // offset = 0; - // memset(buf, 0, bufLimit); - // p = buf; - //} - // if (ttsz >= bufLimit) { - //} char* buf = calloc(1, ttsz * sizeof(char)); char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); tw->ctx->write(tw->ctx, buf, ttsz); - // offset += ttsz; - // p = buf + offset; - // set up value offset v->offset = tw->offset; tw->offset += ttsz; free(buf); } - // if (offset != 0) { - // write reversed data in buf to tindex - // tw->ctx->write(tw->ctx, buf, offset); - //} - // tfree(buf); tw->fb = fstBuilderCreate(tw->ctx, 0); if (tw->fb == NULL) { @@ -331,19 +312,25 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { return 0; } void tfileWriterClose(TFileWriter* tw) { - if (tw == NULL) { return; } + if (tw == NULL) { + return; + } writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { return; } + if (tw == NULL) { + return; + } writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { TFileCache* cache = tfileCacheCreate(path); - if (cache == NULL) { return NULL; } + if (cache == NULL) { + return NULL; + } IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); if (tfile == NULL) { @@ -355,21 +342,27 @@ IndexTFile* indexTFileCreate(const char* path) { return tfile; } void indexTFileDestroy(IndexTFile* tfile) { - if (tfile == NULL) { return; } + if (tfile == NULL) { + return; + } tfileCacheDestroy(tfile->cache); free(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { return ret; } + if (tfile == NULL) { + return ret; + } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { return 0; } + if (reader == NULL) { + return 0; + } return tfileReaderSearch(reader, query, result); } @@ -391,7 +384,9 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { return false; } + if (rt == NULL) { + return false; + } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -401,7 +396,9 @@ static bool tfileIteratorNext(Iterate* iiter) { offset = (uint64_t)(rt->out.out); swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { + return false; + } iv->colVal = colVal; return true; @@ -412,7 +409,9 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { return NULL; } + if (tIter == NULL) { + return NULL; + } tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); @@ -422,7 +421,9 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - if (reader == NULL) { return NULL; } + if (reader == NULL) { + return NULL; + } Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); @@ -437,7 +438,9 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { return; } + if (iter == NULL) { + return; + } IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -452,7 +455,9 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { - if (tf == NULL) { return NULL; } + if (tf == NULL) { + return NULL; + } ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } @@ -464,7 +469,9 @@ static int tfileUidCompare(const void* a, const void* b) { } static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { return ret; } + if (ret == 0) { + return ret; + } return ret < 0 ? -1 : 1; } @@ -479,13 +486,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { return NULL; } + if (tf == NULL) { + return NULL; + } tf->colVal = tstrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { return -1; } + if (tf == NULL) { + return -1; + } taosArrayPush(tf->tableId, &val); return 0; } @@ -506,7 +517,9 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { + return -1; + } tw->offset += sizeof(fstOffset); return 0; } @@ -517,7 +530,9 @@ static int tfileWriteHeader(TFileWriter* writer) { memcpy(buf, (char*)header, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { return -1; } + if (sizeof(buf) != nwrite) { + return -1; + } writer->offset = nwrite; return 0; } @@ -559,7 +574,9 @@ static int tfileReaderLoadFst(TFileReader* reader) { static int FST_MAX_SIZE = 64 * 1024 * 1024; char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { return -1; } + if (buf == NULL) { + return -1; + } WriterCtx* ctx = reader->ctx; int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); @@ -584,23 +601,31 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* int32_t total = sizeof(uint64_t) * nid; char* buf = calloc(1, total); - if (buf == NULL) { return -1; } + if (buf == NULL) { + return -1; + } nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); assert(total == nread); - for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } + for (int32_t i = 0; i < nid; i++) { + taosArrayPush(result, (uint64_t*)buf + i); + } free(buf); return 0; } void tfileReaderRef(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { - if (reader == NULL) { return; } + if (reader == NULL) { + return; + } int ref = T_REF_DEC(reader); if (ref == 0) { // do nothing @@ -616,11 +641,15 @@ static SArray* tfileGetFileList(const char* path) { uint32_t version; DIR* dir = opendir(path); - if (NULL == dir) { return NULL; } + if (NULL == dir) { + return NULL; + } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { char* file = entry->d_name; - if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } + if (0 != tfileParseFileName(file, &suid, buf, &version)) { + continue; + } size_t len = strlen(path) + 1 + strlen(file) + 1; char* buf = calloc(1, len); @@ -643,15 +672,9 @@ static void tfileDestroyFileName(void* elem) { free(p); } static int tfileCompare(const void* a, const void* b) { - const char* aName = *(char**)a; - const char* bName = *(char**)b; - - size_t aLen = strlen(aName); - size_t bLen = strlen(bName); - - int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); - if (ret == 0) { return ret; } - return ret < 0 ? -1 : 1; + const char* as = *(char**)a; + const char* bs = *(char**)b; + return strcmp(as, bs); } static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version) { From 7d29af7316da3f91442fb2fc5735a7195cc807a7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 5 Jan 2022 19:22:06 +0800 Subject: [PATCH 23/54] refactor code --- source/libs/index/src/index.c | 56 +++--------- source/libs/index/src/index_cache.c | 64 ++++---------- source/libs/index/src/index_tfile.c | 128 +++++++--------------------- 3 files changed, 62 insertions(+), 186 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9c66cfe72f..9b7f1c8383 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -63,9 +63,7 @@ static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { // pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { - return -1; - } + if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -75,9 +73,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); - if (sIdx->tindex == NULL) { - goto END; - } + if (sIdx->tindex == NULL) { goto END; } sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; @@ -88,9 +84,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #endif END: - if (sIdx != NULL) { - indexClose(sIdx); - } + if (sIdx != NULL) { indexClose(sIdx); } *index = NULL; return -1; @@ -106,9 +100,7 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { - indexCacheUnRef(*pCache); - } + if (*pCache) { indexCacheUnRef(*pCache); } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); @@ -166,9 +158,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { - return ret; - } + if (ret != 0) { return ret; } } #endif @@ -198,9 +188,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } + for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -257,9 +245,7 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { - return NULL; - } + if (p == NULL) { return NULL; } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -281,9 +267,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { - return NULL; - } + if (t == NULL) { return NULL; } t->suid = suid; t->operType = oper; @@ -356,9 +340,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { - return; - } + if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -414,22 +396,16 @@ static void indexDestroyTempResult(SArray* result) { taosArrayDestroy(result); } int indexFlushCacheTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { - return -1; - } + if (sIdx == NULL) { return -1; } indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); - if (pReader == NULL) { - indexWarn("empty tfile reader found"); - } + if (pReader == NULL) { indexWarn("empty tfile reader found"); } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); - if (tfileIter == NULL) { - indexWarn("empty tfile reader iterator"); - } + if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } SArray* result = taosArrayInit(1024, sizeof(void*)); @@ -496,9 +472,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { taosArrayDestroy(value->val); value->val = NULL; } else { - if (value->val != NULL) { - taosArrayClear(value->val); - } + if (value->val != NULL) { taosArrayClear(value->val); } } free(value->colVal); value->colVal = NULL; @@ -521,9 +495,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { tfileWriterClose(tw); TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); - if (reader == NULL) { - goto END; - } + if (reader == NULL) { goto END; } TFileHeader* header = &reader->header; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index d87c601b00..ca1d9e3fa4 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -115,9 +115,7 @@ void indexCacheDestroySkiplist(SSkipList* slt) { tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); @@ -130,9 +128,7 @@ void indexCacheDestroyImm(IndexCache* cache) { } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { - return; - } + if (pCache == NULL) { return; } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -142,9 +138,7 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { - return NULL; - } + if (iiter == NULL) { return NULL; } pthread_mutex_lock(&cache->mtx); @@ -162,9 +156,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -202,17 +194,13 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -243,9 +231,7 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u } static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { - if (mem == NULL) { - return 0; - } + if (mem == NULL) { return 0; } char* key = getIndexKey(ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); @@ -271,9 +257,7 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA return 0; } int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { - return 0; - } + if (cache == NULL) { return 0; } IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; @@ -301,33 +285,23 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_DEC(cache); - if (ref == 0) { - indexCacheDestroy(cache); - } + if (ref == 0) { indexCacheDestroy(cache); } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -337,9 +311,7 @@ void indexMemUnRef(MemTable* tbl) { } static void cacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { - return; - } + if (ct == NULL) { return; } free(ct->colVal); free(ct); } @@ -354,9 +326,7 @@ static int32_t compareKey(const void* l, const void* r) { // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { - return rt->version - lt->version; - } + if (cmp == 0) { return rt->version - lt->version; } return cmp; } @@ -376,9 +346,7 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { - return false; - } + if (iter == NULL) { return false; } IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 958d511630..6437991358 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -55,9 +55,7 @@ static void tfileGenFileFullName(char* fullname, const char* path, uint64_t s TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { - return NULL; - } + if (tcache == NULL) { return NULL; } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; @@ -73,9 +71,7 @@ TFileCache* tfileCacheCreate(const char* path) { } TFileReader* reader = tfileReaderCreate(wc); - if (reader == NULL) { - goto End; - } + if (reader == NULL) { goto End; } TFileHeader* header = &reader->header; ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName)}; @@ -93,9 +89,7 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { - return; - } + if (tcache == NULL) { return; } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -116,9 +110,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { int32_t sz = indexSerialCacheKey(key, buf); assert(sz < sizeof(buf)); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } tfileReaderRef(*reader); return *reader; @@ -141,9 +133,7 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } // T_REF_INC(reader); reader->ctx = ctx; @@ -163,9 +153,7 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { - return; - } + if (reader == NULL) { return; } // T_REF_INC(reader); fstDestroy(reader->fst); writerCtxDestroy(reader->ctx, reader->remove); @@ -205,9 +193,7 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); // indexInfo("open write file name %s", fullname); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); - if (wcx == NULL) { - return NULL; - } + if (wcx == NULL) { return NULL; } TFileHeader tfh = {0}; tfh.suid = suid; @@ -223,9 +209,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); - if (wc == NULL) { - return NULL; - } + if (wc == NULL) { return NULL; } TFileReader* reader = tfileReaderCreate(wc); return reader; @@ -312,25 +296,19 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { return 0; } void tfileWriterClose(TFileWriter* tw) { - if (tw == NULL) { - return; - } + if (tw == NULL) { return; } writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { - return; - } + if (tw == NULL) { return; } writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { TFileCache* cache = tfileCacheCreate(path); - if (cache == NULL) { - return NULL; - } + if (cache == NULL) { return NULL; } IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); if (tfile == NULL) { @@ -342,27 +320,21 @@ IndexTFile* indexTFileCreate(const char* path) { return tfile; } void indexTFileDestroy(IndexTFile* tfile) { - if (tfile == NULL) { - return; - } + if (tfile == NULL) { return; } tfileCacheDestroy(tfile->cache); free(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { - return ret; - } + if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { - return 0; - } + if (reader == NULL) { return 0; } return tfileReaderSearch(reader, query, result); } @@ -384,9 +356,7 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { - return false; - } + if (rt == NULL) { return false; } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -396,9 +366,7 @@ static bool tfileIteratorNext(Iterate* iiter) { offset = (uint64_t)(rt->out.out); swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { - return false; - } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; return true; @@ -409,9 +377,7 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); - if (tIter == NULL) { - return NULL; - } + if (tIter == NULL) { return NULL; } tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); @@ -421,9 +387,7 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); @@ -438,9 +402,7 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -455,9 +417,7 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { - if (tf == NULL) { - return NULL; - } + if (tf == NULL) { return NULL; } ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } @@ -469,9 +429,7 @@ static int tfileUidCompare(const void* a, const void* b) { } static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } @@ -486,17 +444,13 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { - return NULL; - } + if (tf == NULL) { return NULL; } tf->colVal = tstrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { - return -1; - } + if (tf == NULL) { return -1; } taosArrayPush(tf->tableId, &val); return 0; } @@ -517,9 +471,7 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { - return -1; - } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } tw->offset += sizeof(fstOffset); return 0; } @@ -530,9 +482,7 @@ static int tfileWriteHeader(TFileWriter* writer) { memcpy(buf, (char*)header, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { - return -1; - } + if (sizeof(buf) != nwrite) { return -1; } writer->offset = nwrite; return 0; } @@ -574,9 +524,7 @@ static int tfileReaderLoadFst(TFileReader* reader) { static int FST_MAX_SIZE = 64 * 1024 * 1024; char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); @@ -601,31 +549,23 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* int32_t total = sizeof(uint64_t) * nid; char* buf = calloc(1, total); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); assert(total == nread); - for (int32_t i = 0; i < nid; i++) { - taosArrayPush(result, (uint64_t*)buf + i); - } + for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } free(buf); return 0; } void tfileReaderRef(TFileReader* reader) { - if (reader == NULL) { - return; - } + if (reader == NULL) { return; } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { - if (reader == NULL) { - return; - } + if (reader == NULL) { return; } int ref = T_REF_DEC(reader); if (ref == 0) { // do nothing @@ -641,15 +581,11 @@ static SArray* tfileGetFileList(const char* path) { uint32_t version; DIR* dir = opendir(path); - if (NULL == dir) { - return NULL; - } + if (NULL == dir) { return NULL; } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { char* file = entry->d_name; - if (0 != tfileParseFileName(file, &suid, buf, &version)) { - continue; - } + if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } size_t len = strlen(path) + 1 + strlen(file) + 1; char* buf = calloc(1, len); From ad94dff047eaba6671d4c652daab285ca6e78704 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 04:18:56 -0800 Subject: [PATCH 24/54] mionr changes --- include/common/tmsg.h | 4 +- source/client/src/clientMsgHandler.c | 4 +- source/dnode/mgmt/impl/test/bnode/dbnode.cpp | 54 ++++---- source/dnode/mgmt/impl/test/db/db.cpp | 54 ++++---- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 56 ++++----- source/dnode/mgmt/impl/test/mnode/mnode.cpp | 82 ++++++------ .../dnode/mgmt/impl/test/profile/profile.cpp | 58 ++++----- source/dnode/mgmt/impl/test/qnode/dqnode.cpp | 54 ++++---- source/dnode/mgmt/impl/test/snode/dsnode.cpp | 54 ++++---- source/dnode/mgmt/impl/test/stb/stb.cpp | 32 ++--- source/dnode/mgmt/impl/test/sut/inc/client.h | 2 +- source/dnode/mgmt/impl/test/sut/inc/sut.h | 6 +- .../dnode/mgmt/impl/test/sut/src/client.cpp | 10 +- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 24 ++-- source/dnode/mgmt/impl/test/vgroup/vgroup.cpp | 18 +-- source/dnode/mnode/impl/src/mndAcct.c | 12 +- source/dnode/mnode/impl/src/mndBnode.c | 82 ++++++------ source/dnode/mnode/impl/src/mndQnode.c | 82 ++++++------ source/dnode/mnode/impl/src/mndShow.c | 48 +++---- source/dnode/mnode/impl/src/mndSnode.c | 82 ++++++------ source/dnode/mnode/impl/src/mndUser.c | 62 ++++----- source/dnode/mnode/impl/test/acct/acct.cpp | 28 ++--- source/dnode/mnode/impl/test/bnode/bnode.cpp | 110 ++++++++-------- source/dnode/mnode/impl/test/qnode/qnode.cpp | 110 ++++++++-------- source/dnode/mnode/impl/test/show/show.cpp | 34 ++--- source/dnode/mnode/impl/test/snode/snode.cpp | 110 ++++++++-------- source/dnode/mnode/impl/test/trans/trans.cpp | 16 +-- source/dnode/mnode/impl/test/user/user.cpp | 118 +++++++++--------- source/libs/parser/inc/astToMsg.h | 2 +- source/libs/parser/src/astToMsg.c | 4 +- source/libs/parser/src/dCDAstProcess.c | 2 +- src/client/src/tscServer.c | 18 +-- 32 files changed, 716 insertions(+), 716 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1b2b6a677a..cd8fe7b0d2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -506,7 +506,7 @@ typedef struct { int64_t qId; }; // query handle int8_t free; -} SRetrieveTableMsg; +} SRetrieveTableReq; typedef struct SRetrieveTableRsp { int64_t useconds; @@ -808,7 +808,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; int16_t payloadLen; char payload[]; -} SShowMsg; +} SShowReq; typedef struct { char db[TSDB_DB_FNAME_LEN]; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e402403496..1d74de9ab3 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -87,14 +87,14 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { - SRetrieveTableMsg* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); + SRetrieveTableReq* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableReq)); if (pRetrieveMsg == NULL) { return NULL; } pRetrieveMsg->showId = htobe64(pRequest->body.showInfo.execId); pMsgSendInfo->msgInfo.pData = pRetrieveMsg; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); + pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); } else { SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { diff --git a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp b/source/dnode/mgmt/impl/test/bnode/dbnode.cpp index dd8c13c4bd..bafe8242a6 100644 --- a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/impl/test/bnode/dbnode.cpp @@ -32,9 +32,9 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_BNODE_ID_INVALID); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ID_INVALID); } { @@ -43,9 +43,9 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -54,9 +54,9 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -67,9 +67,9 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); } } @@ -80,9 +80,9 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_BNODE_ID_INVALID); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ID_INVALID); } { @@ -91,9 +91,9 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -102,9 +102,9 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); } test.Restart(); @@ -115,9 +115,9 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_NOT_DEPLOYED); } { @@ -126,8 +126,8 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index a78b8388c6..3a69ae2305 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -26,7 +26,7 @@ class DndTestDb : public ::testing::Test { Testbase DndTestDb::test; TEST_F(DndTestDb, 01_ShowDb) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); @@ -47,7 +47,7 @@ TEST_F(DndTestDb, 01_ShowDb) { CHECK_SCHEMA(16, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); CHECK_SCHEMA(17, TSDB_DATA_TYPE_TINYINT, 1, "update"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } @@ -77,15 +77,15 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); @@ -106,14 +106,14 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("ms", 3); // precision CheckInt8(0); // update - test.SendShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "1.d1"); + test.SendShowMetaReq(TSDB_MGMT_TABLE_VGROUP, "1.d1"); CHECK_META("show vgroups", 4); CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "vgId"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_INT, 4, "tables"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckInt32(2); CheckInt32(3); @@ -138,13 +138,13 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->quorum = 2; pReq->cacheLastRow = 1; - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_DB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); @@ -168,10 +168,10 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { // restart test.Restart(); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); @@ -199,15 +199,15 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_DB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } @@ -237,15 +237,15 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); CHECK_META("show databases", 18); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d2", TSDB_DB_NAME_LEN - 1); @@ -256,7 +256,7 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { strcpy(pReq->db, "1.d2"); pReq->vgVersion = htonl(-1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_USE_DB, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 9041098d71..51af008fc3 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -51,7 +51,7 @@ TestServer DndTestDnode::server4; TestServer DndTestDnode::server5; TEST_F(DndTestDnode, 01_ShowDnode) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); @@ -62,7 +62,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -81,9 +81,9 @@ TEST_F(DndTestDnode, 02_ConfigDnode) { pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONFIG_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONFIG_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { @@ -94,16 +94,16 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9042); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckInt16(1); @@ -127,14 +127,14 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -152,9 +152,9 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9043); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -164,9 +164,9 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9044); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -176,15 +176,15 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9045); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 4); CheckInt16(1); @@ -225,9 +225,9 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { server5.Restart(); taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 4); CheckInt16(1); diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mgmt/impl/test/mnode/mnode.cpp index 3d4844c3f6..9a725c20d3 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/mnode.cpp @@ -51,7 +51,7 @@ TestServer DndTestMnode::server4; TestServer DndTestMnode::server5; TEST_F(DndTestMnode, 01_ShowDnode) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_MNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_MNODE, ""); CHECK_META("show mnodes", 5); CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); @@ -60,7 +60,7 @@ TEST_F(DndTestMnode, 01_ShowDnode) { CHECK_SCHEMA(3, TSDB_DATA_TYPE_TIMESTAMP, 8, "role_time"); CHECK_SCHEMA(4, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -77,9 +77,9 @@ TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MNODE_ALREADY_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MNODE_ALREADY_EXIST); } } @@ -90,9 +90,9 @@ TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); } } @@ -105,13 +105,13 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9062); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); } @@ -122,12 +122,12 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_MNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_MNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckInt16(1); @@ -149,12 +149,12 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { SMDropMnodeMsg* pReq = (SMDropMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_MNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_MNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_MNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -170,14 +170,14 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); // pReq->dnodeId = htonl(2); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_DNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); +// SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); +// ASSERT_NE(pRsp, nullptr); +// ASSERT_EQ(pRsp->code, 0); // } -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); +// test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); // CHECK_META("show dnodes", 7); -// test.SendShowRetrieveMsg(); +// test.SendShowRetrieveReq(); // EXPECT_EQ(test.GetShowRows(), 1); // CheckInt16(1); @@ -194,9 +194,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9063"); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); +// SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); +// ASSERT_NE(pRsp, nullptr); +// ASSERT_EQ(pRsp->code, 0); // } // { @@ -205,9 +205,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9064"); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); +// SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); +// ASSERT_NE(pRsp, nullptr); +// ASSERT_EQ(pRsp->code, 0); // } // { @@ -216,15 +216,15 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9065"); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); +// SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); +// ASSERT_NE(pRsp, nullptr); +// ASSERT_EQ(pRsp->code, 0); // } // taosMsleep(1300); -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); +// test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); // CHECK_META("show dnodes", 7); -// test.SendShowRetrieveMsg(); +// test.SendShowRetrieveReq(); // EXPECT_EQ(test.GetShowRows(), 4); // CheckInt16(1); @@ -265,9 +265,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // server5.Restart(); // taosMsleep(1300); -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); +// test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); // CHECK_META("show dnodes", 7); -// test.SendShowRetrieveMsg(); +// test.SendShowRetrieveReq(); // EXPECT_EQ(test.GetShowRows(), 4); // CheckInt16(1); diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index b705a94af6..29b71f1c27 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -35,7 +35,7 @@ TEST_F(DndTestProfile, 01_ConnectMsg) { strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -67,14 +67,14 @@ TEST_F(DndTestProfile, 02_ConnectMsg_InvalidDB) { strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, "invalid_db"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_DB); - ASSERT_EQ(pMsg->contLen, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DB); + ASSERT_EQ(pRsp->contLen, 0); } TEST_F(DndTestProfile, 03_ConnectMsg_Show) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_CONNS, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); CHECK_META("show connections", 7); CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "connId"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "user"); @@ -84,7 +84,7 @@ TEST_F(DndTestProfile, 03_ConnectMsg_Show) { CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "login_time"); CHECK_SCHEMA(6, TSDB_DATA_TYPE_TIMESTAMP, 8, "last_access"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt32(1); CheckBinary("root", TSDB_USER_LEN); @@ -105,7 +105,7 @@ TEST_F(DndTestProfile, 04_HeartBeatMsg) { pReq->numOfStreams = htonl(0); strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_HEARTBEAT, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -138,9 +138,9 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_CONN, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_CONN, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -153,10 +153,10 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { pReq->numOfStreams = htonl(0); strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_HEARTBEAT, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONNECTION); - ASSERT_EQ(pMsg->contLen, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_HEARTBEAT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_CONNECTION); + ASSERT_EQ(pRsp->contLen, 0); } { @@ -167,7 +167,7 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -198,9 +198,9 @@ TEST_F(DndTestProfile, 06_KillConnMsg_InvalidConn) { SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); pReq->connId = htonl(2345); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_CONN, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_CONN, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_CONN_ID); } TEST_F(DndTestProfile, 07_KillQueryMsg) { @@ -211,10 +211,10 @@ TEST_F(DndTestProfile, 07_KillQueryMsg) { pReq->connId = htonl(connId); pReq->queryId = htonl(1234); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_QUERY, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - ASSERT_EQ(pMsg->contLen, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_QUERY, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + ASSERT_EQ(pRsp->contLen, 0); } { @@ -227,7 +227,7 @@ TEST_F(DndTestProfile, 07_KillQueryMsg) { pReq->numOfStreams = htonl(0); strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_HEARTBEAT, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -261,13 +261,13 @@ TEST_F(DndTestProfile, 08_KillQueryMsg_InvalidConn) { pReq->connId = htonl(2345); pReq->queryId = htonl(1234); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_KILL_QUERY, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_QUERY, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_CONN_ID); } TEST_F(DndTestProfile, 09_KillQueryMsg) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_QUERIES, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_QUERIES, ""); CHECK_META("show queries", 14); CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "queryId"); @@ -285,6 +285,6 @@ TEST_F(DndTestProfile, 09_KillQueryMsg) { CHECK_SCHEMA(12, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, "sub_query_info"); CHECK_SCHEMA(13, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, "sql"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp index 48de6e03eb..e609783eb0 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp @@ -32,9 +32,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_QNODE_ID_INVALID); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ID_INVALID); } { @@ -43,9 +43,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -54,9 +54,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -67,9 +67,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED); } } @@ -80,9 +80,9 @@ TEST_F(DndTestQnode, 01_Drop_Qnode) { SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_QNODE_ID_INVALID); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_ID_INVALID); } { @@ -91,9 +91,9 @@ TEST_F(DndTestQnode, 01_Drop_Qnode) { SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -102,9 +102,9 @@ TEST_F(DndTestQnode, 01_Drop_Qnode) { SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); } test.Restart(); @@ -115,9 +115,9 @@ TEST_F(DndTestQnode, 01_Drop_Qnode) { SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_QNODE_NOT_DEPLOYED); } { @@ -126,8 +126,8 @@ TEST_F(DndTestQnode, 01_Drop_Qnode) { SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/snode/dsnode.cpp b/source/dnode/mgmt/impl/test/snode/dsnode.cpp index 2ec624b942..1e6bcdb3fb 100644 --- a/source/dnode/mgmt/impl/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/impl/test/snode/dsnode.cpp @@ -32,9 +32,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_SNODE_ID_INVALID); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ID_INVALID); } { @@ -43,9 +43,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -54,9 +54,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -67,9 +67,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED); } } @@ -80,9 +80,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_SNODE_ID_INVALID); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_ID_INVALID); } { @@ -91,9 +91,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -102,9 +102,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); } test.Restart(); @@ -115,9 +115,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_SNODE_NOT_DEPLOYED); } { @@ -126,8 +126,8 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index 40af751e33..b3d5d29785 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -51,9 +51,9 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -101,12 +101,12 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { strcpy(pSchema->name, "tag3"); } - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_STB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, "1.d1"); CHECK_META("show stables", 4); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); @@ -114,7 +114,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "columns"); CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "tags"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); @@ -128,7 +128,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(contLen); strcpy(pReq->tableFname, "1.d1.stb"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_STB_META, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -203,9 +203,9 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { // restart test.Restart(); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, "1.d1"); CHECK_META("show stables", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("stb", TSDB_TABLE_NAME_LEN); @@ -219,13 +219,13 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(contLen); strcpy(pReq->name, "1.d1.stb"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_STB, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + test.SendShowMetaReq(TSDB_MGMT_TABLE_STB, "1.d1"); CHECK_META("show stables", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mgmt/impl/test/sut/inc/client.h b/source/dnode/mgmt/impl/test/sut/inc/client.h index e1e430318e..9cf688fc02 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/client.h +++ b/source/dnode/mgmt/impl/test/sut/inc/client.h @@ -21,7 +21,7 @@ class TestClient { bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port); void Cleanup(); - SRpcMsg* SendMsg(SRpcMsg* pMsg); + SRpcMsg* SendReq(SRpcMsg* pReq); void SetRpcRsp(SRpcMsg* pRsp); tsem_t* GetSem(); diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 24278a73f7..fd9c1cd93e 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -39,7 +39,7 @@ class Testbase { void Restart(); void ServerStop(); void ServerStart(); - SRpcMsg* SendMsg(tmsg_t msgType, void* pCont, int32_t contLen); + SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen); private: void InitLog(const char* path); @@ -50,8 +50,8 @@ class Testbase { int32_t connId; public: - void SendShowMetaMsg(int8_t showType, const char* db); - void SendShowRetrieveMsg(); + void SendShowMetaReq(int8_t showType, const char* db); + void SendShowRetrieveReq(); STableMetaMsg* GetShowMeta(); SRetrieveTableRsp* GetRetrieveRsp(); diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index 086ba7bb0f..fd5dcd77d9 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -15,10 +15,10 @@ #include "sut.h" -static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { +static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { TestClient* client = (TestClient*)parent; - client->SetRpcRsp(pMsg); - uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pMsg->msgType), pMsg->code); + client->SetRpcRsp(pRsp); + uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pRsp->msgType), pRsp->code); tsem_post(client->GetSem()); } @@ -59,14 +59,14 @@ void TestClient::Cleanup() { rpcClose(clientRpc); } -SRpcMsg* TestClient::SendMsg(SRpcMsg* pMsg) { +SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) { SEpSet epSet = {0}; epSet.inUse = 0; epSet.numOfEps = 1; epSet.port[0] = port; memcpy(epSet.fqdn[0], fqdn, TSDB_FQDN_LEN); - rpcSendRequest(clientRpc, &epSet, pMsg, NULL); + rpcSendRequest(clientRpc, &epSet, pReq, NULL); tsem_wait(&sem); return pRsp; diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 46ced254c6..72a6416e22 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -64,23 +64,23 @@ void Testbase::ServerStop() { server.Stop(); } void Testbase::ServerStart() { server.DoStart(); } -SRpcMsg* Testbase::SendMsg(tmsg_t msgType, void* pCont, int32_t contLen) { +SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pCont; rpcMsg.contLen = contLen; rpcMsg.msgType = msgType; - return client.SendMsg(&rpcMsg); + return client.SendReq(&rpcMsg); } -void Testbase::SendShowMetaMsg(int8_t showType, const char* db) { - int32_t contLen = sizeof(SShowMsg); - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(contLen); +void Testbase::SendShowMetaReq(int8_t showType, const char* db) { + int32_t contLen = sizeof(SShowReq); + SShowReq* pShow = (SShowReq*)rpcMallocCont(contLen); pShow->type = showType; strcpy(pShow->db, db); - SRpcMsg* pMsg = SendMsg(TDMT_MND_SHOW, pShow, contLen); - SShowRsp* pShowRsp = (SShowRsp*)pMsg->pCont; + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pShow, contLen); + SShowRsp* pShowRsp = (SShowRsp*)pRsp->pCont; ASSERT(pShowRsp != nullptr); pShowRsp->showId = htobe64(pShowRsp->showId); @@ -121,15 +121,15 @@ int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } const char* Testbase::GetMetaTbName() { return pMeta->tbFname; } -void Testbase::SendShowRetrieveMsg() { - int32_t contLen = sizeof(SRetrieveTableMsg); +void Testbase::SendShowRetrieveReq() { + int32_t contLen = sizeof(SRetrieveTableReq); - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(contLen); + SRetrieveTableReq* pRetrieve = (SRetrieveTableReq*)rpcMallocCont(contLen); pRetrieve->showId = htobe64(showId); pRetrieve->free = 0; - SRpcMsg* pMsg = SendMsg(TDMT_MND_SHOW_RETRIEVE, pRetrieve, contLen); - pRetrieveRsp = (SRetrieveTableRsp*)pMsg->pCont; + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pRetrieve, contLen); + pRetrieveRsp = (SRetrieveTableRsp*)pRsp->pCont; pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp index 15a0c8087c..7fa3b4ab61 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -60,9 +60,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { pReplica->port = htons(9150); } - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_CREATE_VNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } @@ -100,9 +100,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { pReplica->port = htons(9150); } - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_ALTER_VNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } @@ -121,9 +121,9 @@ TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { rpcMsg.contLen = sizeof(SDropVnodeMsg); rpcMsg.msgType = TDMT_DND_DROP_VNODE; - SRpcMsg* pMsg = test.SendMsg(TDMT_DND_DROP_VNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 0cf940b987..945a6523ef 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -26,9 +26,9 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw); static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew); -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pMsg); -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pMsg); -static int32_t mndProcessDropAcctReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq); +static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq); +static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq); int32_t mndInitAcct(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_ACCT, @@ -185,19 +185,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { return 0; } -static int32_t mndProcessCreateAcctReq(SMnodeMsg *pMsg) { +static int32_t mndProcessCreateAcctReq(SMnodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessAlterAcctReq(SMnodeMsg *pMsg) { +static int32_t mndProcessAlterAcctReq(SMnodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } -static int32_t mndProcessDropAcctReq(SMnodeMsg *pMsg) { +static int32_t mndProcessDropAcctReq(SMnodeMsg *pReq) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 14d49213e9..6325bbb6fd 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -27,12 +27,12 @@ static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg); -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg); -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg); -static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp); +static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { @@ -185,22 +185,22 @@ static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { } static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDCreateBnodeReq *pMsg = malloc(sizeof(SDCreateBnodeReq)); - if (pMsg == NULL) { + SDCreateBnodeReq *pReq = malloc(sizeof(SDCreateBnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDCreateBnodeReq); action.msgType = TDMT_DND_CREATE_BNODE; action.acceptableCode = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } @@ -208,29 +208,29 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S } static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq *pMsg = malloc(sizeof(SDDropBnodeReq)); - if (pMsg == NULL) { + SDDropBnodeReq *pReq = malloc(sizeof(SDDropBnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDDropBnodeReq); action.msgType = TDMT_DND_DROP_BNODE; action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } return 0; } -static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { +static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { int32_t code = -1; SBnodeObj bnodeObj = {0}; @@ -238,7 +238,7 @@ static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode bnodeObj.createdTime = taosGetTimestampMs(); bnodeObj.updateTime = bnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_BNODE_OVER; mDebug("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); @@ -256,9 +256,9 @@ CREATE_BNODE_OVER: return code; } -static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMCreateBnodeReq *pCreate = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMCreateBnodeReq *pCreate = pReq->rpcMsg.pCont; pCreate->dnodeId = htonl(pCreate->dnodeId); @@ -282,7 +282,7 @@ static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateBnode(pMnode, pMsg, pDnode, pCreate); + int32_t code = mndCreateBnode(pMnode, pReq, pDnode, pCreate); mndReleaseDnode(pMnode, pDnode); if (code != 0) { @@ -310,32 +310,32 @@ static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { } static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq *pMsg = malloc(sizeof(SDDropBnodeReq)); - if (pMsg == NULL) { + SDDropBnodeReq *pReq = malloc(sizeof(SDDropBnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDDropBnodeReq); action.msgType = TDMT_DND_DROP_BNODE; action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } return 0; } -static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pMsg, SBnodeObj *pObj) { +static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pReq, SBnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_BNODE_OVER; mDebug("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); @@ -351,9 +351,9 @@ DROP_BNODE_OVER: return code; } -static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMDropBnodeReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMDropBnodeReq *pDrop = pReq->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("bnode:%d, start to drop", pDrop->dnodeId); @@ -370,7 +370,7 @@ static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropBnode(pMnode, pMsg, pObj); + int32_t code = mndDropBnode(pMnode, pReq, pObj); if (code != 0) { sdbRelease(pMnode->pSdb, pObj); mError("bnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); @@ -381,18 +381,18 @@ static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -431,8 +431,8 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveBnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 6951aa8717..a223890e8c 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -27,12 +27,12 @@ static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg); -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg); -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg); -static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp); +static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); int32_t mndInitQnode(SMnode *pMnode) { @@ -185,22 +185,22 @@ static int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { } static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { - SDCreateQnodeReq *pMsg = malloc(sizeof(SDCreateQnodeReq)); - if (pMsg == NULL) { + SDCreateQnodeReq *pReq = malloc(sizeof(SDCreateQnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDCreateQnodeReq); action.msgType = TDMT_DND_CREATE_QNODE; action.acceptableCode = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } @@ -208,29 +208,29 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S } static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { - SDDropQnodeReq *pMsg = malloc(sizeof(SDDropQnodeReq)); - if (pMsg == NULL) { + SDDropQnodeReq *pReq = malloc(sizeof(SDDropQnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDDropQnodeReq); action.msgType = TDMT_DND_DROP_QNODE; action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } return 0; } -static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { +static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateQnodeReq *pCreate) { int32_t code = -1; SQnodeObj qnodeObj = {0}; @@ -238,7 +238,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode qnodeObj.createdTime = taosGetTimestampMs(); qnodeObj.updateTime = qnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_QNODE_OVER; mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); @@ -256,9 +256,9 @@ CREATE_QNODE_OVER: return code; } -static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMCreateQnodeReq *pCreate = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMCreateQnodeReq *pCreate = pReq->rpcMsg.pCont; pCreate->dnodeId = htonl(pCreate->dnodeId); @@ -282,7 +282,7 @@ static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateQnode(pMnode, pMsg, pDnode, pCreate); + int32_t code = mndCreateQnode(pMnode, pReq, pDnode, pCreate); mndReleaseDnode(pMnode, pDnode); if (code != 0) { @@ -310,32 +310,32 @@ static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { } static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { - SDDropQnodeReq *pMsg = malloc(sizeof(SDDropQnodeReq)); - if (pMsg == NULL) { + SDDropQnodeReq *pReq = malloc(sizeof(SDDropQnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDDropQnodeReq); action.msgType = TDMT_DND_DROP_QNODE; action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } return 0; } -static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pMsg, SQnodeObj *pObj) { +static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_QNODE_OVER; mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); @@ -351,9 +351,9 @@ DROP_QNODE_OVER: return code; } -static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMDropQnodeReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMDropQnodeReq *pDrop = pReq->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("qnode:%d, start to drop", pDrop->dnodeId); @@ -370,7 +370,7 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropQnode(pMnode, pMsg, pObj); + int32_t code = mndDropQnode(pMnode, pReq, pObj); if (code != 0) { sdbRelease(pMnode->pSdb, pObj); mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); @@ -381,18 +381,18 @@ static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -431,8 +431,8 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveQnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 81e15497bf..9c145f6445 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -18,12 +18,12 @@ #define SHOW_STEP_SIZE 100 -static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); +static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static int32_t mndProcessShowReq(SMnodeMsg *pMsg); -static int32_t mndProcessRetrieveReq(SMnodeMsg *pMsg); +static int32_t mndProcessShowReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq); static bool mndCheckRetrieveFinished(SShowObj *pShow); int32_t mndInitShow(SMnode *pMnode) { @@ -49,20 +49,20 @@ void mndCleanupShow(SMnode *pMnode) { } } -static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { +static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) { SShowMgmt *pMgmt = &pMnode->showMgmt; int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1); if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1); - int32_t size = sizeof(SShowObj) + pMsg->payloadLen; + int32_t size = sizeof(SShowObj) + pReq->payloadLen; SShowObj showObj = {0}; showObj.id = showId; showObj.pMnode = pMnode; - showObj.type = pMsg->type; - showObj.payloadLen = pMsg->payloadLen; - memcpy(showObj.db, pMsg->db, TSDB_DB_FNAME_LEN); - memcpy(showObj.payload, pMsg->payload, pMsg->payloadLen); + showObj.type = pReq->type; + showObj.payloadLen = pReq->payloadLen; + memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); + memcpy(showObj.payload, pReq->payload, pReq->payloadLen); int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); @@ -115,12 +115,12 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { taosCacheRelease(pMgmt->cache, (void **)(&pShow), forceRemove); } -static int32_t mndProcessShowReq(SMnodeMsg *pMnodeMsg) { - SMnode *pMnode = pMnodeMsg->pMnode; +static int32_t mndProcessShowReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont; - int8_t type = pMsg->type; - int16_t payloadLen = htonl(pMsg->payloadLen); + SShowReq *pShowReq = pReq->rpcMsg.pCont; + int8_t type = pShowReq->type; + int16_t payloadLen = htonl(pShowReq->payloadLen); if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; @@ -135,7 +135,7 @@ static int32_t mndProcessShowReq(SMnodeMsg *pMnodeMsg) { return -1; } - SShowObj *pShow = mndCreateShowObj(pMnode, pMsg); + SShowObj *pShow = mndCreateShowObj(pMnode, pShowReq); if (pShow == NULL) { mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); return -1; @@ -151,13 +151,13 @@ static int32_t mndProcessShowReq(SMnodeMsg *pMnodeMsg) { return -1; } - int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta); + int32_t code = (*metaFp)(pReq, pShow, &pRsp->tableMeta); mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d type:%s, result:%s", pShow->id, pShow->numOfRows, pShow->numOfColumns, mndShowStr(type), tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { - pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; - pMnodeMsg->pCont = pRsp; + pReq->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; + pReq->pCont = pRsp; pRsp->showId = htobe64(pShow->id); mndReleaseShowObj(pShow, false); return TSDB_CODE_SUCCESS; @@ -168,14 +168,14 @@ static int32_t mndProcessShowReq(SMnodeMsg *pMnodeMsg) { } } -static int32_t mndProcessRetrieveReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; int32_t rowsRead = 0; - SRetrieveTableMsg *pRetrieve = pMsg->rpcMsg.pCont; + SRetrieveTableReq *pRetrieve = pReq->rpcMsg.pCont; int64_t showId = htobe64(pRetrieve->showId); SShowObj *pShow = mndAcquireShowObj(pMnode, showId); @@ -227,7 +227,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pMsg) { // if free flag is set, client wants to clean the resources if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { - rowsRead = (*retrieveFp)(pMsg, pShow, pRsp->data, rowsToRead); + rowsRead = (*retrieveFp)(pReq, pShow, pRsp->data, rowsToRead); } mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); @@ -235,8 +235,8 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pMsg) { pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision - pMsg->pCont = pRsp; - pMsg->contLen = size; + pReq->pCont = pRsp; + pReq->contLen = size; if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 7221a2fbf4..1b66bedbcc 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -27,12 +27,12 @@ static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg); -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg); -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg); -static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp); +static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { @@ -185,22 +185,22 @@ static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { } static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { - SDCreateSnodeReq *pMsg = malloc(sizeof(SDCreateSnodeReq)); - if (pMsg == NULL) { + SDCreateSnodeReq *pReq = malloc(sizeof(SDCreateSnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDCreateSnodeReq); action.msgType = TDMT_DND_CREATE_SNODE; action.acceptableCode = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } @@ -208,29 +208,29 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S } static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { - SDDropSnodeReq *pMsg = malloc(sizeof(SDDropSnodeReq)); - if (pMsg == NULL) { + SDDropSnodeReq *pReq = malloc(sizeof(SDDropSnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDDropSnodeReq); action.msgType = TDMT_DND_DROP_SNODE; action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } return 0; } -static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { +static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode, SMCreateSnodeReq *pCreate) { int32_t code = -1; SSnodeObj snodeObj = {0}; @@ -238,7 +238,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode snodeObj.createdTime = taosGetTimestampMs(); snodeObj.updateTime = snodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_SNODE_OVER; mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); @@ -257,9 +257,9 @@ CREATE_SNODE_OVER: return code; } -static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMCreateSnodeReq *pCreate = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMCreateSnodeReq *pCreate = pReq->rpcMsg.pCont; pCreate->dnodeId = htonl(pCreate->dnodeId); @@ -283,7 +283,7 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateSnode(pMnode, pMsg, pDnode, pCreate); + int32_t code = mndCreateSnode(pMnode, pReq, pDnode, pCreate); mndReleaseDnode(pMnode, pDnode); if (code != 0) { @@ -311,32 +311,32 @@ static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { } static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { - SDDropSnodeReq *pMsg = malloc(sizeof(SDDropSnodeReq)); - if (pMsg == NULL) { + SDDropSnodeReq *pReq = malloc(sizeof(SDDropSnodeReq)); + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pDnode->id); + pReq->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pMsg; + action.pCont = pReq; action.contLen = sizeof(SDDropSnodeReq); action.msgType = TDMT_DND_DROP_SNODE; action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); + free(pReq); return -1; } return 0; } -static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pMsg, SSnodeObj *pObj) { +static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg); if (pTrans == NULL) goto DROP_SNODE_OVER; mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); @@ -353,9 +353,9 @@ DROP_SNODE_OVER: return code; } -static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMDropSnodeReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMDropSnodeReq *pDrop = pReq->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("snode:%d, start to drop", pDrop->dnodeId); @@ -372,7 +372,7 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropSnode(pMnode, pMsg, pObj); + int32_t code = mndDropSnode(pMnode, pReq, pObj); if (code != 0) { sdbRelease(pMnode->pSdb, pObj); mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); @@ -383,18 +383,18 @@ static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -433,8 +433,8 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveSnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 0891cf43b3..e73b34e688 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -28,12 +28,12 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew); -static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg); -static int32_t mndProcessCreateUserReq(SMnodeMsg *pMsg); -static int32_t mndProcessAlterUserReq(SMnodeMsg *pMsg); -static int32_t mndProcessDropUserReq(SMnodeMsg *pMsg); -static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq); +static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq); +static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq); +static int32_t mndProcessDropUserReq(SMnodeMsg *pReq); +static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { @@ -212,7 +212,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { sdbRelease(pSdb, pUser); } -static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg) { +static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) { SUserObj userObj = {0}; tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); @@ -221,7 +221,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, userObj.updateTime = userObj.createdTime; userObj.superUser = 0; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to create since %s", user, terrstr()); return -1; @@ -246,9 +246,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, return 0; } -static int32_t mndProcessCreateUserReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SCreateUserReq *pCreate = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SCreateUserReq *pCreate = pReq->rpcMsg.pCont; mDebug("user:%s, start to create", pCreate->user); @@ -272,14 +272,14 @@ static int32_t mndProcessCreateUserReq(SMnodeMsg *pMsg) { return -1; } - SUserObj *pOperUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; mError("user:%s, failed to create since %s", pCreate->user, terrstr()); return -1; } - int32_t code = mndCreateUser(pMnode, pOperUser->acct, pCreate->user, pCreate->pass, pMsg); + int32_t code = mndCreateUser(pMnode, pOperUser->acct, pCreate->user, pCreate->pass, pReq); mndReleaseUser(pMnode, pOperUser); if (code != 0) { @@ -290,8 +290,8 @@ static int32_t mndProcessCreateUserReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SMnodeMsg *pMsg) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SMnodeMsg *pReq) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to update since %s", pOld->user, terrstr()); return -1; @@ -316,9 +316,9 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SMn return 0; } -static int32_t mndProcessAlterUserReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SAlterUserReq *pAlter = pMsg->rpcMsg.pCont; +static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SAlterUserReq *pAlter = pReq->rpcMsg.pCont; mDebug("user:%s, start to alter", pAlter->user); @@ -341,7 +341,7 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pMsg) { return -1; } - SUserObj *pOperUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; @@ -355,7 +355,7 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pMsg) { taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); newUser.updateTime = taosGetTimestampMs(); - int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg); + int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq); mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pUser); @@ -367,8 +367,8 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); +static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pReq, SUserObj *pUser) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); return -1; @@ -393,9 +393,9 @@ static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { return 0; } -static int32_t mndProcessDropUserReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropUserReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SDropUserReq *pDrop = pReq->rpcMsg.pCont; mDebug("user:%s, start to drop", pDrop->user); @@ -412,7 +412,7 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pMsg) { return -1; } - SUserObj *pOperUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { mndReleaseUser(pMnode, pUser); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; @@ -420,7 +420,7 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropUser(pMnode, pMsg, pUser); + int32_t code = mndDropUser(pMnode, pReq, pUser); mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pUser); @@ -432,8 +432,8 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -478,8 +478,8 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p return 0; } -static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveUsers(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SUserObj *pUser = NULL; diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 906a066f74..315f23f798 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -30,9 +30,9 @@ TEST_F(MndTestAcct, 01_Create_Acct) { SCreateAcctReq* pReq = (SCreateAcctReq*)rpcMallocCont(contLen); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_ACCT, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(MndTestAcct, 02_Alter_Acct) { @@ -40,9 +40,9 @@ TEST_F(MndTestAcct, 02_Alter_Acct) { SAlterAcctReq* pReq = (SAlterAcctReq*)rpcMallocCont(contLen); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_ACCT, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(MndTestAcct, 03_Drop_Acct) { @@ -50,18 +50,18 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { SDropAcctReq* pReq = (SDropAcctReq*)rpcMallocCont(contLen); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_ACCT, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(MndTestAcct, 04_Show_Acct) { - int32_t contLen = sizeof(SShowMsg); + int32_t contLen = sizeof(SShowReq); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); + SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); pReq->type = TSDB_MGMT_TABLE_ACCT; - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_SHOW, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index 07dc163df5..e32c0e7f91 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -39,14 +39,14 @@ Testbase MndTestBnode::test; TestServer MndTestBnode::server2; TEST_F(MndTestBnode, 01_Show_Bnode) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_BNODE, ""); CHECK_META("show bnodes", 3); CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } @@ -57,9 +57,9 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); } { @@ -68,13 +68,13 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_BNODE, ""); CHECK_META("show bnodes", 3); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -88,9 +88,9 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_BNODE_ALREADY_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_BNODE_ALREADY_EXIST); } } @@ -102,13 +102,13 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9019); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); } @@ -118,12 +118,12 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckInt16(1); @@ -140,12 +140,12 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -159,9 +159,9 @@ TEST_F(MndTestBnode, 03_Drop_Bnode) { SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_BNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_BNODE_NOT_EXIST); } } @@ -174,9 +174,9 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } { @@ -187,9 +187,9 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); } { @@ -200,9 +200,9 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); } { @@ -219,9 +219,9 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - if (pMsg->code == 0) break; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; taosMsleep(1000); } @@ -238,9 +238,9 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } { @@ -251,9 +251,9 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); } { @@ -264,9 +264,9 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); } { @@ -283,9 +283,9 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - if (pMsg->code == 0) break; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; taosMsleep(1000); } diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 612b138ccc..55806dca4d 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -39,14 +39,14 @@ Testbase MndTestQnode::test; TestServer MndTestQnode::server2; TEST_F(MndTestQnode, 01_Show_Qnode) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); CHECK_META("show qnodes", 3); CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } @@ -57,9 +57,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); } { @@ -68,13 +68,13 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); CHECK_META("show qnodes", 3); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -88,9 +88,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_QNODE_ALREADY_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_QNODE_ALREADY_EXIST); } } @@ -102,13 +102,13 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9015); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); } @@ -118,12 +118,12 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckInt16(1); @@ -140,12 +140,12 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -159,9 +159,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_QNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_QNODE_NOT_EXIST); } } @@ -174,9 +174,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } { @@ -187,9 +187,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); } { @@ -200,9 +200,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); } { @@ -219,9 +219,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - if (pMsg->code == 0) break; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; taosMsleep(1000); } @@ -238,9 +238,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } { @@ -251,9 +251,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); } { @@ -264,9 +264,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); } { @@ -283,9 +283,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - if (pMsg->code == 0) break; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; taosMsleep(1000); } diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 6d2c9150f2..b4a625d8be 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -26,27 +26,27 @@ class MndTestShow : public ::testing::Test { Testbase MndTestShow::test; TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { - int32_t contLen = sizeof(SShowMsg); + int32_t contLen = sizeof(SShowReq); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); + SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); pReq->type = TSDB_MGMT_TABLE_MAX; strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_SHOW, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { - int32_t contLen = sizeof(SShowMsg); + int32_t contLen = sizeof(SShowReq); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + SShowReq* pReq = (SShowReq*)rpcMallocCont(sizeof(SShowReq)); pReq->type = TSDB_MGMT_TABLE_START; strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_SHOW, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } TEST_F(MndTestShow, 03_ShowMsg_Conn) { @@ -57,11 +57,11 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { strcpy(pReq->app, "mnode_test_show"); strcpy(pReq->db, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CONNECT, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_CONNS, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); STableMetaMsg* pMeta = test.GetShowMeta(); EXPECT_STREQ(pMeta->tbFname, "show connections"); @@ -75,7 +75,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { EXPECT_EQ(pMeta->tuid, 0); EXPECT_EQ(pMeta->suid, 0); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); SRetrieveTableRsp* pRetrieveRsp = test.GetRetrieveRsp(); EXPECT_EQ(pRetrieveRsp->numOfRows, 1); @@ -87,13 +87,13 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { } TEST_F(MndTestShow, 04_ShowMsg_Cluster) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_CLUSTER, ""); CHECK_META( "show cluster", 3); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BIGINT, 8, "id"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); IgnoreInt64(); diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index cebb5fe113..9e07be842f 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -39,14 +39,14 @@ Testbase MndTestSnode::test; TestServer MndTestSnode::server2; TEST_F(MndTestSnode, 01_Show_Snode) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_SNODE, ""); CHECK_META("show snodes", 3); CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 0); } @@ -57,9 +57,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); } { @@ -68,13 +68,13 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_SNODE, ""); CHECK_META("show snodes", 3); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -88,9 +88,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_SNODE_ALREADY_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_SNODE_ALREADY_EXIST); } } @@ -102,13 +102,13 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9017); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); taosMsleep(1300); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); } @@ -118,12 +118,12 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckInt16(1); @@ -140,12 +140,12 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); - test.SendShowRetrieveMsg(); + test.SendShowMetaReq(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); @@ -159,9 +159,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_SNODE_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_SNODE_NOT_EXIST); } } @@ -174,9 +174,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } { @@ -187,9 +187,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); } { @@ -200,9 +200,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_CREATING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); } { @@ -219,9 +219,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - if (pMsg->code == 0) break; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; taosMsleep(1000); } @@ -238,9 +238,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); } { @@ -251,9 +251,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); } { @@ -264,9 +264,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { pReq->dnodeId = htonl(2); server2.Stop(); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_SDB_OBJ_DROPPING); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); } { @@ -283,9 +283,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - if (pMsg->code == 0) break; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; taosMsleep(1000); } diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 97a8bd2caf..37c426f573 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -58,21 +58,21 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); KillThenRestartServer(); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckBinary("u1", TSDB_USER_LEN); @@ -83,4 +83,4 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { CheckTimestamp(); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 76954db213..a696158d2b 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -26,7 +26,7 @@ class MndTestUser : public ::testing::Test { Testbase MndTestUser::test; TEST_F(MndTestUser, 01_Show_User) { - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); @@ -34,7 +34,7 @@ TEST_F(MndTestUser, 01_Show_User) { CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("root", TSDB_USER_LEN); @@ -51,9 +51,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(pReq->user, ""); strcpy(pReq->pass, "p1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_USER_FORMAT); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_USER_FORMAT); } { @@ -63,9 +63,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_PASS_FORMAT); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_PASS_FORMAT); } { @@ -75,9 +75,9 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(pReq->user, "root"); strcpy(pReq->pass, "1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_USER_ALREADY_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_USER_ALREADY_EXIST); } { @@ -87,15 +87,15 @@ TEST_F(MndTestUser, 02_Create_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); } @@ -107,9 +107,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(pReq->user, ""); strcpy(pReq->pass, "p1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_USER_FORMAT); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_USER_FORMAT); } { @@ -119,9 +119,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_PASS_FORMAT); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_PASS_FORMAT); } { @@ -131,9 +131,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(pReq->user, "u4"); strcpy(pReq->pass, "1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_USER_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_USER_NOT_EXIST); } { @@ -143,9 +143,9 @@ TEST_F(MndTestUser, 03_Alter_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } } @@ -156,9 +156,9 @@ TEST_F(MndTestUser, 04_Drop_User) { SDropUserReq* pReq = (SDropUserReq*)rpcMallocCont(contLen); strcpy(pReq->user, ""); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_USER_FORMAT); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_USER_FORMAT); } { @@ -167,9 +167,9 @@ TEST_F(MndTestUser, 04_Drop_User) { SDropUserReq* pReq = (SDropUserReq*)rpcMallocCont(contLen); strcpy(pReq->user, "u4"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_USER_NOT_EXIST); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_USER_NOT_EXIST); } { @@ -178,15 +178,15 @@ TEST_F(MndTestUser, 04_Drop_User) { SDropUserReq* pReq = (SDropUserReq*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 1); } @@ -198,9 +198,9 @@ TEST_F(MndTestUser, 05_Create_Drop_Alter_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } { @@ -210,15 +210,15 @@ TEST_F(MndTestUser, 05_Create_Drop_Alter_User) { strcpy(pReq->user, "u2"); strcpy(pReq->pass, "p2"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 3); CheckBinary("u1", TSDB_USER_LEN); @@ -241,15 +241,15 @@ TEST_F(MndTestUser, 05_Create_Drop_Alter_User) { strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_ALTER_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 3); CheckBinary("u1", TSDB_USER_LEN); @@ -271,15 +271,15 @@ TEST_F(MndTestUser, 05_Create_Drop_Alter_User) { SDropUserReq* pReq = (SDropUserReq*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); - SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_USER, pReq, contLen); - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_USER, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); } - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckBinary("root", TSDB_USER_LEN); @@ -294,10 +294,10 @@ TEST_F(MndTestUser, 05_Create_Drop_Alter_User) { // restart test.Restart(); - test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); CHECK_META("show users", 4); - test.SendShowRetrieveMsg(); + test.SendShowRetrieveReq(); EXPECT_EQ(test.GetShowRows(), 2); CheckBinary("root", TSDB_USER_LEN); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 30919d25dd..bbb7731261 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -8,7 +8,7 @@ SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowMsg* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index ab8c9b8094..81181c34dc 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -85,8 +85,8 @@ SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, cha return pMsg; } -SShowMsg* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, int32_t msgLen) { - SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, int32_t msgLen) { + SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6007fc300c..faaa9924f6 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -109,7 +109,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou *pEpSet = pCtx->mgmtEpSet; *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len); - *outputLen = sizeof(SShowMsg) /* + htons(pShowMsg->payloadLen)*/; + *outputLen = sizeof(SShowReq) /* + htons(pShowMsg->payloadLen)*/; } return TSDB_CODE_SUCCESS; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 48127147f8..e0489a76d2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -603,7 +603,7 @@ int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { } int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; + SRetrieveTableReq *pRetrieveMsg = (SRetrieveTableReq *) pSql->cmd.payload; SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); @@ -638,10 +638,10 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->res.qId); } - pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); + pSql->cmd.payloadLen = sizeof(SRetrieveTableReq); pSql->cmd.msgType = TDMT_VND_FETCH; - pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); + pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableReq)); return TSDB_CODE_SUCCESS; } @@ -1420,7 +1420,7 @@ int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TDMT_MND_SHOW; - pCmd->payloadLen = sizeof(SShowMsg) + 100; + pCmd->payloadLen = sizeof(SShowReq) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); @@ -1428,13 +1428,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; - SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; + SShowReq *pShowMsg = (SShowReq *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { pShowMsg->type = pShowInfo->showType; pShowMsg->payloadLen = 0; - pCmd->payloadLen = sizeof(SShowMsg); + pCmd->payloadLen = sizeof(SShowReq); return TSDB_CODE_SUCCESS; } @@ -1463,7 +1463,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pEpAddr->n); } - pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + pCmd->payloadLen = sizeof(SShowReq) + htons(pShowMsg->payloadLen); return TSDB_CODE_SUCCESS; } @@ -1742,7 +1742,7 @@ int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TDMT_MND_SHOW_RETRIEVE; - pCmd->payloadLen = sizeof(SRetrieveTableMsg); + pCmd->payloadLen = sizeof(SRetrieveTableReq); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); @@ -1750,7 +1750,7 @@ int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; + SRetrieveTableReq *pRetrieveMsg = (SRetrieveTableReq*)pCmd->payload; pRetrieveMsg->qId = htobe64(pSql->res.qId); pRetrieveMsg->free = htons(pQueryInfo->type); From 3c3f32bdae4aa21aa8e37e162817a229269a64f2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 05:17:16 -0800 Subject: [PATCH 25/54] minor changes --- source/dnode/mgmt/impl/test/qnode/dqnode.cpp | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 21 +++++----- source/dnode/mnode/impl/test/trans/trans.cpp | 42 +++++++++++++++++++- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp index e609783eb0..e64a0543fc 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp @@ -73,7 +73,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { } } -TEST_F(DndTestQnode, 01_Drop_Qnode) { +TEST_F(DndTestQnode, 02_Drop_Qnode) { { int32_t contLen = sizeof(SDDropQnodeReq); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c1686c2923..11bd15d270 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -24,7 +24,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); @@ -314,9 +314,10 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { return 0; } -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { - mTrace("trans:%d, perform update action, old_row:%p new_row:%p", pOldTrans->id, pOldTrans, pNewTrans); - pOldTrans->stage = pNewTrans->stage; +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { + mTrace("trans:%d, perform update action, old row:%p stage:%d, new row:%p stage:%d", pOld->id, pOld, pOld->stage, pNew, + pNew->stage); + pOld->stage = pNew->stage; return 0; } @@ -464,16 +465,16 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { } mDebug("trans:%d, prepare finished", pTrans->id); - STrans *pNewTrans = mndAcquireTrans(pMnode, pTrans->id); - if (pNewTrans == NULL) { + STrans *pNew = mndAcquireTrans(pMnode, pTrans->id); + if (pNew == NULL) { mError("trans:%d, failed to read from sdb since %s", pTrans->id, terrstr()); return -1; } - pNewTrans->rpcHandle = pTrans->rpcHandle; - pNewTrans->rpcAHandle = pTrans->rpcAHandle; - mndTransExecute(pMnode, pNewTrans); - mndReleaseTrans(pMnode, pNewTrans); + pNew->rpcHandle = pTrans->rpcHandle; + pNew->rpcAHandle = pTrans->rpcAHandle; + mndTransExecute(pMnode, pNew); + mndReleaseTrans(pMnode, pNew); return 0; } diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 37c426f573..eef94436b7 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -10,7 +10,6 @@ */ #include "sut.h" -#include "os.h" class MndTestTrans : public ::testing::Test { protected: @@ -84,3 +83,44 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); } + +TEST_F(MndTestTrans, 02_Create_Qnode_Crash) { + { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + KillThenRestartServer(); + { + int32_t retry = 0; + int32_t retryMax = 10; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; + taosMsleep(1000); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } +} \ No newline at end of file From 2e0dc58890c07708074bfe29a022a60a9573ae4e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 5 Jan 2022 22:56:08 +0800 Subject: [PATCH 26/54] refactor code --- source/libs/index/inc/index_cache.h | 2 +- source/libs/index/src/index_cache.c | 13 ++++++++----- source/libs/index/src/index_tfile.c | 3 --- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 805137ccaf..58082294a9 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -40,7 +40,7 @@ typedef struct IndexCache { SIndex* index; char* colName; int32_t version; - int32_t nTerm; + int64_t occupiedMem; int8_t type; uint64_t suid; diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index ca1d9e3fa4..7a4beb2119 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -21,6 +21,8 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 +#define MEM_THRESHOLD 1024 * 1024 * 2 +#define MEM_ESTIMATE_RADIO 1.5 static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); @@ -50,6 +52,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in cache->index = idx; cache->version = 0; cache->suid = suid; + cache->occupiedMem = 0; pthread_mutex_init(&cache->mtx, NULL); indexCacheRef(cache); return cache; @@ -173,8 +176,7 @@ int indexCacheSchedToMerge(IndexCache* pCache) { } static void indexCacheMakeRoomForWrite(IndexCache* cache) { while (true) { - if (cache->nTerm < MEM_TERM_LIMIT) { - cache->nTerm += 1; + if (cache->occupiedMem * MEM_ESTIMATE_RADIO < MEM_THRESHOLD) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable @@ -185,7 +187,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); - cache->nTerm = 1; + cache->occupiedMem = 0; // sched to merge // unref cache in bgwork indexCacheSchedToMerge(cache); @@ -211,8 +213,9 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { ct->operaType = term->operType; // ugly code, refactor later + int64_t estimate = sizeof(ct) + strlen(ct->colVal); pthread_mutex_lock(&pCache->mtx); - + pCache->occupiedMem += estimate; indexCacheMakeRoomForWrite(pCache); MemTable* tbl = pCache->mem; indexMemRef(tbl); @@ -333,7 +336,7 @@ static int32_t compareKey(const void* l, const void* r) { static MemTable* indexInternalCacheCreate(int8_t type) { MemTable* tbl = calloc(1, sizeof(MemTable)); indexMemRef(tbl); - if (type == TSDB_DATA_TYPE_BINARY) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); } return tbl; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 6437991358..90a730d3a9 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -346,9 +346,6 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { } static bool tfileIteratorNext(Iterate* iiter) { IterateValue* iv = &iiter->val; - if (iv->colVal != NULL && iv->val != NULL) { - // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); - } iterateValueDestroy(iv, false); char* colVal = NULL; From ccbbac4eefdc79bef9c6387d7c54e7adc4c46b52 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 5 Jan 2022 23:54:05 +0800 Subject: [PATCH 27/54] refactor code --- source/libs/index/inc/index_cache.h | 1 + source/libs/index/src/index.c | 17 +++++++++++++++-- source/libs/index/src/index_cache.c | 14 +++++++++++--- source/libs/index/test/indexTests.cc | 6 +++--- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 58082294a9..544849ff2f 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -45,6 +45,7 @@ typedef struct IndexCache { uint64_t suid; pthread_mutex_t mtx; + pthread_cond_t finished; } IndexCache; #define CACHE_VERSION(cache) atomic_load_32(&cache->version) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9b7f1c8383..0bf30d6efa 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -399,6 +399,8 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { if (sIdx == NULL) { return -1; } indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + int64_t st = taosGetTimestampUs(); + IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); if (pReader == NULL) { indexWarn("empty tfile reader found"); } @@ -458,6 +460,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { } int ret = indexGenTFile(sIdx, pCache, result); indexDestroyTempResult(result); + indexCacheDestroyImm(pCache); indexCacheIteratorDestroy(cacheIter); @@ -465,7 +468,14 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { tfileReaderUnRef(pReader); indexCacheUnRef(pCache); - return 0; + + int64_t cost = taosGetTimestampUs() - st; + if (ret != 0) { + indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000); + } else { + indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); + } + return ret; } void iterateValueDestroy(IterateValue* value, bool destroy) { if (destroy) { @@ -506,7 +516,10 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { pthread_mutex_unlock(&sIdx->mtx); return ret; END: - tfileWriterClose(tw); + if (tw != NULL) { + writerCtxDestroy(tw->ctx, true); + free(tw); + } return -1; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 7a4beb2119..d43c9f9cce 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -53,7 +53,10 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in cache->version = 0; cache->suid = suid; cache->occupiedMem = 0; + pthread_mutex_init(&cache->mtx, NULL); + pthread_cond_init(&cache->finished, NULL); + indexCacheRef(cache); return cache; } @@ -124,6 +127,7 @@ void indexCacheDestroyImm(IndexCache* cache) { pthread_mutex_lock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread + pthread_cond_broadcast(&cache->finished); pthread_mutex_unlock(&cache->mtx); indexMemUnRef(tbl); @@ -136,6 +140,9 @@ void indexCacheDestroy(void* cache) { indexMemUnRef(pCache->imm); free(pCache->colName); + pthread_mutex_destroy(&pCache->mtx); + pthread_cond_destroy(&pCache->finished); + free(pCache); } @@ -180,9 +187,10 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - pthread_mutex_unlock(&cache->mtx); - taosMsleep(50); - pthread_mutex_lock(&cache->mtx); + // pthread_mutex_unlock(&cache->mtx); + pthread_cond_wait(&cache->finished, &cache->mtx); + // taosMsleep(50); + // pthread_mutex_lock(&cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 9c92af26a2..5438f88b76 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -28,7 +28,7 @@ #include "tutil.h" using namespace std; -#define NUM_OF_THREAD 5 +#define NUM_OF_THREAD 10 class DebugInfo { public: @@ -882,8 +882,8 @@ static void single_write_and_search(IndexObj* idx) { static void multi_write_and_search(IndexObj* idx) { int target = idx->SearchOne("tag1", "Hello"); target = idx->SearchOne("tag2", "Test"); - idx->WriteMultiMillonData("tag1", "Hello", 100 * 10000); - idx->WriteMultiMillonData("tag2", "Test", 100 * 10000); + idx->WriteMultiMillonData("tag1", "hello world test", 100 * 10000); + idx->WriteMultiMillonData("tag2", "world test nothing", 100 * 10000); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { std::string path = "/tmp/cache_and_tfile"; From 308faffccd9dd33237fba50816a0ab21da420c88 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 6 Jan 2022 10:18:17 +0800 Subject: [PATCH 28/54] [TD-12758]add unit test in 3.0 CI --- Jenkinsfile | 221 +----------------- .../dnode/mgmt/impl/test/profile/profile.cpp | 2 +- 2 files changed, 3 insertions(+), 220 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c47d37a9a0..ef6e7d67b2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,228 +109,11 @@ pipeline { sh''' cd ${WKC}/tests ./test-all.sh b1fq + cd ${WKC}/debug + ctest ''' } } - // stage('Parallel test stage') { - // skip defaultCheckout - // options { skipDefaultCheckout() } - // when { - // allOf{ - // changeRequest() - - // } - // } - // parallel { - // stage('python_1_s1') { - // agent{label " slave1 || slave11 "} - // steps { - - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh p1 - // // date''' - // // } - - // } - // } - // stage('python_2_s5') { - // agent{label " slave5 || slave15 "} - // steps { - - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh p2 - // // date''' - // // } - // } - // } - // stage('python_3_s6') { - // agent{label " slave6 || slave16 "} - // steps { - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh p3 - // // date''' - // // } - // } - // } - // stage('test_b1_s2') { - // agent{label " slave2 || slave12 "} - // steps { - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - - // // sh ''' - // // rm -rf /var/lib/taos/* - // // rm -rf /var/log/taos/* - // // nohup taosd >/dev/null & - // // sleep 10 - // // ''' - // // sh ''' - // // cd ${WKC}/tests/examples/nodejs - // // npm install td2.0-connector > /dev/null 2>&1 - // // node nodejsChecker.js host=localhost - // // node test1970.js - // // cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport - // // npm install td2.0-connector > /dev/null 2>&1 - // // node nanosecondTest.js - - // // ''' - // // sh ''' - // // cd ${WKC}/tests/examples/C#/taosdemo - // // mcs -out:taosdemo *.cs > /dev/null 2>&1 - // // echo '' |./taosdemo -c /etc/taos - // // cd ${WKC}/tests/connectorTest/C#Test/nanosupport - // // mcs -out:nano *.cs > /dev/null 2>&1 - // // echo '' |./nano - // // ''' - // // sh ''' - // // cd ${WKC}/tests/gotest - // // bash batchtest.sh - // // ''' - // // sh ''' - // // cd ${WKC}/tests - // // ./test-all.sh b1fq - // // date''' - // // } - // } - // } - // stage('test_crash_gen_s3') { - // agent{label " slave3 || slave13 "} - - // steps { - // pre_test() - // // timeout(time: 60, unit: 'MINUTES'){ - // // sh ''' - // // cd ${WKC}/tests/pytest - // // ./crash_gen.sh -a -p -t 4 -s 2000 - // // ''' - // // } - // // timeout(time: 60, unit: 'MINUTES'){ - // // // sh ''' - // // // cd ${WKC}/tests/pytest - // // // rm -rf /var/lib/taos/* - // // // rm -rf /var/log/taos/* - // // // ./handle_crash_gen_val_log.sh - // // // ''' - // // sh ''' - // // cd ${WKC}/tests/pytest - // // rm -rf /var/lib/taos/* - // // rm -rf /var/log/taos/* - // // ./handle_taosd_val_log.sh - // // ''' - // // } - // // timeout(time: 55, unit: 'MINUTES'){ - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh b2fq - // // date - // // ''' - // // } - // } - // } - // stage('test_valgrind_s4') { - // agent{label " slave4 || slave14 "} - - // steps { - // pre_test() - // // catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - // // sh ''' - // // cd ${WKC}/tests/pytest - // // ./valgrind-test.sh 2>&1 > mem-error-out.log - // // ./handle_val_log.sh - // // ''' - // // } - // // timeout(time: 55, unit: 'MINUTES'){ - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh b3fq - // // date''' - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh full example - // // date''' - // // } - // } - // } - // stage('test_b4_s7') { - // agent{label " slave7 || slave17 "} - // steps { - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh b4fq - // // cd ${WKC}/tests - // // ./test-all.sh p4 - // // cd ${WKC}/tests - // // ./test-all.sh full jdbc - // // cd ${WKC}/tests - // // ./test-all.sh full unit - // // date''' - // // } - // } - // } - // stage('test_b5_s8') { - // agent{label " slave8 || slave18 "} - // steps { - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh b5fq - // // date''' - // // } - // } - // } - // stage('test_b6_s9') { - // agent{label " slave9 || slave19 "} - // steps { - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh b6fq - // // date''' - // // } - // } - // } - // stage('test_b7_s10') { - // agent{label " slave10 || slave20 "} - // steps { - // pre_test() - // // timeout(time: 55, unit: 'MINUTES'){ - - // // sh ''' - // // date - // // cd ${WKC}/tests - // // ./test-all.sh b7fq - // // date''' - // // } - // } - // } - // } - // } } post { success { diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 29b71f1c27..48bd0d1bbf 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -117,7 +117,7 @@ TEST_F(DndTestProfile, 04_HeartBeatMsg) { pRsp->totalDnodes = htonl(pRsp->totalDnodes); pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); - + EXPECT_EQ(1,0); EXPECT_EQ(pRsp->connId, connId); EXPECT_EQ(pRsp->queryId, 0); EXPECT_EQ(pRsp->streamId, 0); From d7fc6e17f99753c6e8f5d0666b322139e7a3b770 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 6 Jan 2022 10:26:22 +0800 Subject: [PATCH 29/54] revert --- source/dnode/mgmt/impl/test/profile/profile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 48bd0d1bbf..29b71f1c27 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -117,7 +117,7 @@ TEST_F(DndTestProfile, 04_HeartBeatMsg) { pRsp->totalDnodes = htonl(pRsp->totalDnodes); pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); - EXPECT_EQ(1,0); + EXPECT_EQ(pRsp->connId, connId); EXPECT_EQ(pRsp->queryId, 0); EXPECT_EQ(pRsp->streamId, 0); From 0d4169102caf196ce3dc3003f9bd07033924fd4f Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 6 Jan 2022 10:27:00 +0800 Subject: [PATCH 30/54] udpate --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index ef6e7d67b2..23dc54d963 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,6 +109,8 @@ pipeline { sh''' cd ${WKC}/tests ./test-all.sh b1fq + ''' + sh''' cd ${WKC}/debug ctest ''' From 0ba5427a9bef40ee25b047c9302732600c30dcd9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 6 Jan 2022 10:28:34 +0800 Subject: [PATCH 31/54] handle subscribe --- include/common/tmsgdef.h | 1 + include/util/taoserror.h | 2 + source/dnode/mnode/impl/inc/mndConsumer.h | 7 +- source/dnode/mnode/impl/inc/mndDef.h | 85 +++-- source/dnode/mnode/impl/inc/mndTopic.h | 7 +- source/dnode/mnode/impl/src/mndConsumer.c | 416 +++++++++++++--------- source/dnode/mnode/impl/src/mndTopic.c | 105 +++--- source/libs/wal/src/walMeta.c | 3 + source/libs/wal/src/walRead.c | 13 +- source/libs/wal/src/walSeek.c | 9 +- source/libs/wal/src/walWrite.c | 2 + 11 files changed, 385 insertions(+), 265 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index d6071ebcf3..592672b32b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -159,6 +159,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ae36ac7216..a10dd8eda2 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -400,6 +400,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") +#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) //"WAL invalid version") +#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) //"WAL out of memory") // tfs #define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 60f186d7d2..68ba08b66e 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -25,11 +25,8 @@ extern "C" { int32_t mndInitConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode); -SConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); -void mndReleaseConsumer(SMnode *pMnode, SConsumerObj *pConsumer); - -SCGroupObj *mndAcquireCGroup(SMnode *pMnode, char *consumerGroup); -void mndReleaseCGroup(SMnode *pMnode, SCGroupObj *pCGroup); +SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); +void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1af9b840a9..b490eeac96 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -302,6 +302,7 @@ typedef struct { char payload[]; } SShowObj; +#if 0 typedef struct SConsumerObj { uint64_t uid; int64_t createTime; @@ -309,17 +310,59 @@ typedef struct SConsumerObj { //uint64_t dbUid; int32_t version; SRWLatch lock; - SList* topics; + SArray* topics; } SConsumerObj; +typedef struct SMqTopicConsumer { + int64_t consumerId; + SList* topicList; +} SMqTopicConsumer; +#endif + +typedef struct SMqCGroup { + char name[TSDB_CONSUMER_GROUP_LEN]; + int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal + SList *consumerIds; // SList + SList *idleVGroups; // SList +} SMqCGroup; + +typedef struct SMqTopicObj { + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + uint64_t uid; + uint64_t dbUid; + int32_t version; + SRWLatch lock; + int32_t sqlLen; + char *sql; + char *logicalPlan; + char *physicalPlan; + SHashObj *cgroups; // SHashObj +} SMqTopicObj; + +// TODO: add cache and change name to id +typedef struct SMqConsumerTopic { + char name[TSDB_TOPIC_FNAME_LEN]; + SList *vgroups; // SList +} SMqConsumerTopic; + +typedef struct SMqConsumerObj { + SRWLatch lock; + int64_t consumerId; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; + SArray *topics; // SArray +} SMqConsumerObj; + typedef struct SMqSubConsumerObj { int64_t consumerUid; // if -1, unassigned - SList* vgId; //SList + SList *vgId; // SList } SMqSubConsumerObj; typedef struct SMqSubCGroupObj { char name[TSDB_CONSUMER_GROUP_LEN]; - SList* consumers; //SList + SList *consumers; // SList } SMqSubCGroupObj; typedef struct SMqSubTopicObj { @@ -332,32 +375,33 @@ typedef struct SMqSubTopicObj { int32_t version; SRWLatch lock; int32_t sqlLen; - char* sql; - char* logicalPlan; - char* physicalPlan; - SList* cgroups; //SList + char *sql; + char *logicalPlan; + char *physicalPlan; + SList *cgroups; // SList } SMqSubTopicObj; typedef struct SMqConsumerSubObj { int64_t topicUid; - SList* vgIds; //SList + SList *vgIds; // SList } SMqConsumerSubObj; typedef struct SMqConsumerHbObj { int64_t consumerId; - SList* consumerSubs; //SList + SList *consumerSubs; // SList } SMqConsumerHbObj; typedef struct SMqVGroupSubObj { int64_t topicUid; - SList* consumerIds; //SList + SList *consumerIds; // SList } SMqVGroupSubObj; typedef struct SMqVGroupHbObj { int64_t vgId; - SList* vgSubs; //SList + SList *vgSubs; // SList } SMqVGroupHbObj; +#if 0 typedef struct SCGroupObj { char name[TSDB_TOPIC_FNAME_LEN]; int64_t createTime; @@ -368,24 +412,7 @@ typedef struct SCGroupObj { SRWLatch lock; SList* consumerIds; } SCGroupObj; - -typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - uint64_t uid; - uint64_t dbUid; - int32_t version; - SRWLatch lock; - int32_t execLen; - void* executor; - int32_t sqlLen; - char* sql; - char* logicalPlan; - char* physicalPlan; - SList* consumerIds; -} STopicObj; +#endif typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index d092f47d4b..fd82c60d37 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -25,8 +25,11 @@ extern "C" { int32_t mndInitTopic(SMnode *pMnode); void mndCleanupTopic(SMnode *pMnode); -STopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName); -void mndReleaseTopic(SMnode *pMnode, STopicObj *pTopic); +SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName); +void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); + +SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); +SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5391b0f73e..eeb011c084 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -24,16 +24,17 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "tcompare.h" #include "tname.h" #define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_RESERVE_SIZE 64 -static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer); +static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); -static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer); -static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer); -static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pConsumer, SConsumerObj *pNewConsumer); +static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); +static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); +static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg); @@ -57,8 +58,8 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq); + /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/ + /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/ mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); return sdbSetTable(pMnode->pSdb, table); @@ -66,85 +67,41 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} -static SSdbRaw *mndCGroupActionEncode(SCGroupObj *pCGroup) { - int32_t size = sizeof(SConsumerObj) + MND_CONSUMER_RESERVE_SIZE; +static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { + return 0; +} + +static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { + int32_t size = sizeof(SMqConsumerObj) + MND_CONSUMER_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) return NULL; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pCGroup->name, TSDB_TABLE_FNAME_LEN); - SDB_SET_INT64(pRaw, dataPos, pCGroup->createTime); - SDB_SET_INT64(pRaw, dataPos, pCGroup->updateTime); - SDB_SET_INT64(pRaw, dataPos, pCGroup->uid); - /*SDB_SET_INT64(pRaw, dataPos, pConsumer->dbUid);*/ - SDB_SET_INT32(pRaw, dataPos, pCGroup->version); - - int32_t sz = listNEles(pCGroup->consumerIds); - SDB_SET_INT32(pRaw, dataPos, sz); - - SListIter iter; - tdListInitIter(pCGroup->consumerIds, &iter, TD_LIST_FORWARD); - SListNode *pn = NULL; - while ((pn = tdListNext(&iter)) != NULL) { - int64_t consumerId = *(int64_t *)pn->data; - SDB_SET_INT64(pRaw, dataPos, consumerId); + int32_t topicNum = taosArrayGetSize(pConsumer->topics); + SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId); + int32_t len = strlen(pConsumer->cgroup); + SDB_SET_INT32(pRaw, dataPos, len); + SDB_SET_BINARY(pRaw, dataPos, pConsumer->cgroup, len); + SDB_SET_INT32(pRaw, dataPos, topicNum); + for (int i = 0; i < topicNum; i++) { + int32_t len; + SMqConsumerTopic *pConsumerTopic = taosArrayGet(pConsumer->topics, i); + len = strlen(pConsumerTopic->name); + SDB_SET_INT32(pRaw, dataPos, len); + SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len); + int vgSize; + if (pConsumerTopic->vgroups == NULL) { + vgSize = 0; + } else { + vgSize = listNEles(pConsumerTopic->vgroups); + } + SDB_SET_INT32(pRaw, dataPos, vgSize); + for (int j = 0; j < vgSize; j++) { + // SList* head; + /*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/ + } } - SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE); - SDB_SET_DATALEN(pRaw, dataPos); - return pRaw; -} - -static SSdbRow *mndCGroupActionDecode(SSdbRaw *pRaw) { - int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - - if (sver != MND_CONSUMER_VER_NUMBER) { - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - mError("failed to decode cgroup since %s", terrstr()); - return NULL; - } - - // TODO: maximum size is not known - int32_t size = sizeof(SCGroupObj) + 128 * sizeof(int64_t); - SSdbRow *pRow = sdbAllocRow(size); - SCGroupObj *pCGroup = sdbGetRowObj(pRow); - if (pCGroup == NULL) return NULL; - - int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, pRow, dataPos, pCGroup->name, TSDB_TABLE_FNAME_LEN); - SDB_GET_INT64(pRaw, pRow, dataPos, &pCGroup->createTime); - SDB_GET_INT64(pRaw, pRow, dataPos, &pCGroup->updateTime); - SDB_GET_INT64(pRaw, pRow, dataPos, &pCGroup->uid); - /*SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->dbUid);*/ - SDB_GET_INT32(pRaw, pRow, dataPos, &pCGroup->version); - - int32_t sz; - SDB_GET_INT32(pRaw, pRow, dataPos, &sz); - // TODO: free list when failing - tdListInit(pCGroup->consumerIds, sizeof(int64_t)); - for (int i = 0; i < sz; i++) { - int64_t consumerId; - SDB_GET_INT64(pRaw, pRow, dataPos, &consumerId); - tdListAppend(pCGroup->consumerIds, &consumerId); - } - - SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_CONSUMER_RESERVE_SIZE); - return pRow; -} - -static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer) { - int32_t size = sizeof(SConsumerObj) + MND_CONSUMER_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); - if (pRaw == NULL) return NULL; - - int32_t dataPos = 0; - SDB_SET_INT64(pRaw, dataPos, pConsumer->uid); - SDB_SET_INT64(pRaw, dataPos, pConsumer->createTime); - SDB_SET_INT64(pRaw, dataPos, pConsumer->updateTime); - /*SDB_SET_INT64(pRaw, dataPos, pConsumer->dbUid);*/ - SDB_SET_INT32(pRaw, dataPos, pConsumer->version); - SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE); SDB_SET_DATALEN(pRaw, dataPos); @@ -161,56 +118,67 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { return NULL; } - int32_t size = sizeof(SConsumerObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); - SConsumerObj *pConsumer = sdbGetRowObj(pRow); + int32_t size = sizeof(SMqConsumerObj); + SSdbRow *pRow = sdbAllocRow(size); + SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); if (pConsumer == NULL) return NULL; int32_t dataPos = 0; - SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->uid); - SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->createTime); - SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->updateTime); - /*SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->dbUid);*/ - SDB_GET_INT32(pRaw, pRow, dataPos, &pConsumer->version); + SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->consumerId); + int32_t len, topicNum; + SDB_GET_INT32(pRaw, pRow, dataPos, &len); + SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumer->cgroup, len); + SDB_GET_INT32(pRaw, pRow, dataPos, &topicNum); + for (int i = 0; i < topicNum; i++) { + int32_t topicLen; + SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); + if (pConsumerTopic == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + // TODO + return NULL; + } + /*pConsumerTopic->vgroups = taosArrayInit(topicNum, sizeof(SMqConsumerTopic));*/ + SDB_GET_INT32(pRaw, pRow, dataPos, &topicLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumerTopic->name, topicLen); + int32_t vgSize; + SDB_GET_INT32(pRaw, pRow, dataPos, &vgSize); + } SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_CONSUMER_RESERVE_SIZE); return pRow; } -static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer) { - mTrace("consumer:%ld, perform insert action", pConsumer->uid); +static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { + mTrace("consumer:%ld, perform insert action", pConsumer->consumerId); return 0; } -static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer) { - mTrace("consumer:%ld, perform delete action", pConsumer->uid); +static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { + mTrace("consumer:%ld, perform delete action", pConsumer->consumerId); return 0; } -static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pOldConsumer, SConsumerObj *pNewConsumer) { - mTrace("consumer:%ld, perform update action", pOldConsumer->uid); - atomic_exchange_32(&pOldConsumer->updateTime, pNewConsumer->updateTime); - atomic_exchange_32(&pOldConsumer->version, pNewConsumer->version); - - taosWLockLatch(&pOldConsumer->lock); +static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { + mTrace("consumer:%ld, perform update action", pOldConsumer->consumerId); // TODO handle update + /*taosWLockLatch(&pOldConsumer->lock);*/ + /*taosWUnLockLatch(&pOldConsumer->lock);*/ - taosWUnLockLatch(&pOldConsumer->lock); return 0; } -SConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) { - SSdb *pSdb = pMnode->pSdb; - SConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); +SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) { + SSdb *pSdb = pMnode->pSdb; + SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); if (pConsumer == NULL) { /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ } return pConsumer; } -void mndReleaseConsumer(SMnode *pMnode, SConsumerObj *pConsumer) { +void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pConsumer); } @@ -220,48 +188,185 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { char *msgStr = pMsg->rpcMsg.pCont; SCMSubscribeReq *pSubscribe; tDeserializeSCMSubscribeReq(msgStr, pSubscribe); - int topicNum = pSubscribe->topicNum; int64_t consumerId = pSubscribe->consumerId; char *consumerGroup = pSubscribe->consumerGroup; - // get consumer group and add client into it - SCGroupObj *pCGroup = sdbAcquire(pMnode->pSdb, SDB_CGROUP, consumerGroup); - if (pCGroup != NULL) { - // iterate the list until finding the consumer - // add consumer to cgroup list if not found - // put new record + + SArray *newSub = NULL; + int newTopicNum = pSubscribe->topicNum; + if (newTopicNum) { + newSub = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); + } + for (int i = 0; i < newTopicNum; i++) { + char *topic = pSubscribe->topicName[i]; + SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); + if (pConsumerTopic == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + // TODO: free + return -1; + } + pConsumerTopic->vgroups = tdListNew(sizeof(int64_t)); + taosArrayPush(newSub, pConsumerTopic); + free(pConsumerTopic); + } + taosArraySortString(newSub, taosArrayCompareString); + + SArray *oldSub = NULL; + int oldTopicNum = 0; + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + if (pConsumer == NULL) { + // create consumer + pConsumer = malloc(sizeof(SMqConsumerObj)); + if (pConsumer == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + strcpy(pConsumer->cgroup, pSubscribe->consumerGroup); + + } else { + oldSub = pConsumer->topics; + oldTopicNum = taosArrayGetSize(oldSub); + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + return -1; } - SConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pConsumer != NULL) { - //reset topic list - } - - for (int i = 0; i < topicNum; i++) { - char *topicName = pSubscribe->topicName[i]; - STopicObj *pTopic = mndAcquireTopic(pMnode, topicName); - //get - // consumer id - SList *list = pTopic->consumerIds; - // add the consumer if not in the list - // - SList* topicList = pConsumer->topics; - //add to topic + int i = 0, j = 0; + while (i < newTopicNum || j < oldTopicNum) { + SMqConsumerTopic *pOldTopic = NULL; + SMqConsumerTopic *pNewTopic = NULL; + if (i >= newTopicNum) { + // encode unset topic msg to all vnodes related to that topic + pOldTopic = taosArrayGet(oldSub, j); + j++; + } else if (j >= oldTopicNum) { + pNewTopic = taosArrayGet(newSub, i); + } else { + pNewTopic = taosArrayGet(newSub, i); + pOldTopic = taosArrayGet(oldSub, j); + + char *newName = pNewTopic->name; + char *oldName = pOldTopic->name; + int comp = compareLenPrefixedStr(newName, oldName); + if (comp == 0) { + // do nothing + pOldTopic = pNewTopic = NULL; + i++; + j++; + continue; + } else if (comp < 0) { + pOldTopic = NULL; + i++; + } else { + pNewTopic = NULL; + j++; + } + } + + if (pOldTopic != NULL) { + ASSERT(pNewTopic == NULL); + char *oldTopicName = pOldTopic->name; + SList *vgroups = pOldTopic->vgroups; + SListIter iter; + tdListInitIter(vgroups, &iter, TD_LIST_FORWARD); + SListNode *pn; + + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName); + ASSERT(pTopic != NULL); + SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup)); + while ((pn = tdListNext(&iter)) != NULL) { + int32_t vgId = *(int64_t *)pn->data; + SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); + // TODO release + if (pVgObj == NULL) { + // TODO handle error + continue; + } + // acquire and get epset + void *pMqVgSetReq = + mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, pSubscribe->consumerId, pSubscribe->consumerGroup); + // TODO:serialize + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + action.pCont = pMqVgSetReq; + action.contLen = 0; // TODO + action.msgType = TDMT_VND_MQ_SET_CONN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMqVgSetReq); + mndTransDrop(pTrans); + // TODO free + return -1; + } + } + taosHashRemove(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup)); + + } else if (pNewTopic != NULL) { + ASSERT(pOldTopic == NULL); + + char *newTopicName = pNewTopic->name; + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); + ASSERT(pTopic != NULL); + + SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup)); + if (pGroup == NULL) { + // add new group + pGroup = malloc(sizeof(SMqCGroup)); + if (pGroup == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pGroup->consumerIds = tdListNew(sizeof(int64_t)); + if (pGroup->consumerIds == NULL) { + free(pGroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pGroup->status = 0; + // add into cgroups + taosHashPut(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup), pGroup, + sizeof(SMqCGroup)); + } + + // put the consumer into list + // rebalance will be triggered by timer + tdListAppend(pGroup->consumerIds, &pSubscribe->consumerId); + + SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic); + sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY); + // TODO: error handling + mndTransAppendRedolog(pTrans, pTopicRaw); + } else { + ASSERT(0); + } + } + // destroy old sub + taosArrayDestroy(oldSub); + // put new sub into consumerobj + pConsumer->topics = newSub; + + // persist consumerObj + SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); + sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); + // TODO: error handling + mndTransAppendRedolog(pTrans, pConsumerRaw); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; } + // TODO: free memory + mndTransDrop(pTrans); return 0; } -static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg) { return 0; } - static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); - return 0; -} - static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; @@ -339,7 +444,7 @@ static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumO int32_t numOfConsumers = 0; void *pIter = NULL; while (1) { - SConsumerObj *pConsumer = NULL; + SMqConsumerObj *pConsumer = NULL; pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); if (pIter == NULL) break; @@ -402,49 +507,6 @@ static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMs return 0; } -static int32_t mndRetrieveCGroup(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SCGroupObj *pCGroup = NULL; - int32_t cols = 0; - char *pWrite; - char prefix[64] = {0}; - - tstrncpy(prefix, pShow->db, 64); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = (int32_t)strlen(prefix); - - while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pCGroup); - if (pShow->pIter == NULL) break; - - if (strncmp(pCGroup->name, prefix, prefixLen) != 0) { - sdbRelease(pSdb, pCGroup); - continue; - } - - cols = 0; - - char consumerName[TSDB_TABLE_NAME_LEN] = {0}; - tstrncpy(consumerName, pCGroup->name + prefixLen, TSDB_TABLE_NAME_LEN); - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, consumerName); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pCGroup->createTime; - cols++; - - numOfRows++; - sdbRelease(pSdb, pCGroup); - } - - pShow->numOfReads += numOfRows; - mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); - return numOfRows; -} - static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 24e32e07b4..4f6567a50a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "mndTopic.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -27,18 +28,16 @@ #define MND_TOPIC_VER_NUMBER 1 #define MND_TOPIC_RESERVE_SIZE 64 -static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic); -static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); -static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic); -static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic); -static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pTopic, STopicObj *pNewTopic); -static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); +static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); +static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); +static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); +static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_TOPIC, @@ -58,8 +57,9 @@ int32_t mndInitTopic(SMnode *pMnode) { void mndCleanupTopic(SMnode *pMnode) {} -static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) { - int32_t size = sizeof(STopicObj) + MND_TOPIC_RESERVE_SIZE; +SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { + int32_t len; + int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); if (pRaw == NULL) return NULL; @@ -71,10 +71,15 @@ static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) { SDB_SET_INT64(pRaw, dataPos, pTopic->uid); SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid); SDB_SET_INT32(pRaw, dataPos, pTopic->version); - SDB_SET_INT32(pRaw, dataPos, pTopic->execLen); - SDB_SET_BINARY(pRaw, dataPos, pTopic->executor, pTopic->execLen); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen); + SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen); + len = strlen(pTopic->logicalPlan); + SDB_SET_INT32(pRaw, dataPos, len); + SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len); + len = strlen(pTopic->physicalPlan); + SDB_SET_INT32(pRaw, dataPos, len); + SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len); SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE); SDB_SET_DATALEN(pRaw, dataPos); @@ -82,7 +87,7 @@ static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) { return pRaw; } -static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { +SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; @@ -92,11 +97,12 @@ static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { return NULL; } - int32_t size = sizeof(STopicObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); - STopicObj *pTopic = sdbGetRowObj(pRow); + int32_t size = sizeof(SMqTopicObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + SMqTopicObj *pTopic = sdbGetRowObj(pRow); if (pTopic == NULL) return NULL; + int32_t len; int32_t dataPos = 0; SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN); SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->db, TSDB_DB_FNAME_LEN); @@ -105,49 +111,51 @@ static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->uid); SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->dbUid); SDB_GET_INT32(pRaw, pRow, dataPos, &pTopic->version); - SDB_GET_INT32(pRaw, pRow, dataPos, &pTopic->execLen); - SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->executor, pTopic->execLen); SDB_GET_INT32(pRaw, pRow, dataPos, &pTopic->sqlLen); SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->sql, pTopic->sqlLen); + SDB_GET_INT32(pRaw, pRow, dataPos, &len); + SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->logicalPlan, len); + SDB_GET_INT32(pRaw, pRow, dataPos, &len); + SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->physicalPlan, len); SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_TOPIC_RESERVE_SIZE); return pRow; } -static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic) { +static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic) { mTrace("topic:%s, perform insert action", pTopic->name); return 0; } -static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic) { +static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { mTrace("topic:%s, perform delete action", pTopic->name); return 0; } -static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pOldTopic, STopicObj *pNewTopic) { +static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) { mTrace("topic:%s, perform update action", pOldTopic->name); atomic_exchange_32(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->version, pNewTopic->version); taosWLockLatch(&pOldTopic->lock); - - //TODO handle update + + // TODO handle update taosWUnLockLatch(&pOldTopic->lock); return 0; } -STopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) { - SSdb *pSdb = pMnode->pSdb; - STopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); +SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) { + SSdb *pSdb = pMnode->pSdb; + SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); if (pTopic == NULL) { terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; } return pTopic; } -void mndReleaseTopic(SMnode *pMnode, STopicObj *pTopic) { +void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pTopic); } @@ -162,7 +170,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { return mndAcquireDb(pMnode, db); } -static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { +static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicMsg); SDDropTopicMsg *pDrop = calloc(1, contLen); @@ -185,7 +193,7 @@ static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *pCreate) { } static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { - STopicObj topicObj = {0}; + SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); topicObj.createTime = taosGetTimestampMs(); @@ -197,13 +205,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); if (pTopicRaw == NULL) return -1; if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; + // TODO: replace with trans to support recovery return sdbWrite(pMnode->pSdb, pTopicRaw); } static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - char *msgStr = pMsg->rpcMsg.pCont; - SCMCreateTopicReq* pCreate; + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SCMCreateTopicReq *pCreate; tDeserializeSCMCreateTopicReq(msgStr, pCreate); mDebug("topic:%s, start to create", pCreate->name); @@ -213,7 +222,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { return -1; } - STopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name); if (pTopic != NULL) { sdbRelease(pMnode->pSdb, pTopic); if (pCreate->igExists) { @@ -245,9 +254,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, STopicObj *pTopic) { - return 0; -} +static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; } static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; @@ -255,7 +262,7 @@ static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) { mDebug("topic:%s, start to drop", pDrop->name); - STopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name); if (pTopic == NULL) { if (pDrop->igNotExists) { mDebug("topic:%s, not exist, ignore not exist is set", pDrop->name); @@ -361,7 +368,7 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo int32_t numOfTopics = 0; void *pIter = NULL; while (1) { - STopicObj *pTopic = NULL; + SMqTopicObj *pTopic = NULL; pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); if (pIter == NULL) break; @@ -440,13 +447,13 @@ static void mndExtractTableName(char *tableId, char *name) { } static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - STopicObj *pTopic = NULL; - int32_t cols = 0; - char *pWrite; - char prefix[64] = {0}; + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SMqTopicObj *pTopic = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; tstrncpy(prefix, pShow->db, 64); strcat(prefix, TS_PATH_DELIMITER); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 0f155f9553..6164ec2baf 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -261,15 +261,18 @@ int walLoadMeta(SWal* pWal) { memset(buf, 0, size + 5); int tfd = tfOpenRead(fnameStr); if (tfRead(tfd, buf, size) != size) { + tfClose(tfd); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); if (code != 0) { + tfClose(tfd); free(buf); return -1; } + tfClose(tfd); free(buf); return 0; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index c80fb4eed8..67c8694fd6 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -15,6 +15,7 @@ #include "tfile.h" #include "walInt.h" +#include "taoserror.h" SWalReadHandle *walOpenReadHandle(SWal *pWal) { SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); @@ -30,6 +31,7 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { pRead->status = 0; pRead->pHead = malloc(sizeof(SWalHead)); if (pRead->pHead == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; free(pRead); return NULL; } @@ -55,16 +57,19 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); code = tfLseek(idxTfd, offset, SEEK_SET); if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } // TODO:deserialize ASSERT(entry.ver == ver); code = tfLseek(logTfd, entry.offset, SEEK_SET); if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } return code; @@ -79,6 +84,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); int64_t logTfd = tfOpenRead(fnameStr); if (logTfd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -100,6 +106,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } if (ver < pWal->vers.snapshotVer) { @@ -113,7 +120,6 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { if (pRead->curFileFirstVer != pRet->firstVer) { code = walReadChangeFile(pRead, pRet->firstVer); if (code < 0) { - // TODO: set error flag return -1; } } @@ -132,7 +138,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { // TODO: check wal life if (pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); - if (code != 0) { + if (code < 0) { return -1; } } @@ -145,11 +151,13 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } code = walValidHeadCksum(pRead->pHead); if (code != 0) { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } if (pRead->capacity < pRead->pHead->head.len) { void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } pRead->pHead = ptr; @@ -163,6 +171,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } pRead->curVersion++; diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1f9b7b6e58..1d9f7bdf4d 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -30,17 +30,20 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = tfLseek(idxTfd, idxOff, SEEK_SET); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; // TODO:deserialize code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ASSERT(entry.ver == ver); code = tfLseek(logTfd, entry.offset, SEEK_CUR); if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } return code; @@ -56,11 +59,13 @@ int walChangeFileToLast(SWal* pWal) { walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenReadWrite(fnameStr); if (idxTfd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenReadWrite(fnameStr); if (logTfd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } // switch file @@ -76,11 +81,12 @@ int walChangeFile(SWal* pWal, int64_t ver) { code = tfClose(pWal->writeLogTfd); if (code != 0) { // TODO + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } code = tfClose(pWal->writeIdxTfd); if (code != 0) { - // TODO + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalFileInfo tmpInfo; @@ -113,6 +119,7 @@ int walSeekVer(SWal* pWal, int64_t ver) { return 0; } if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } if (ver < pWal->vers.snapshotVer) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c8ffd9d07d..5e26dd8278 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -25,6 +25,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) { ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { + terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } pWal->vers.commitVer = ver; @@ -38,6 +39,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return 0; } if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { + terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } pthread_mutex_lock(&pWal->mutex); From b0d935ba7da7f15175c7206629444e90085d5f8c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 6 Jan 2022 10:56:25 +0800 Subject: [PATCH 32/54] merge from 3.0 --- include/common/tmsg.h | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 41 ++++++++++++----------- source/dnode/mnode/impl/src/mndTopic.c | 41 +++++++++++------------ 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 177a3b4b1e..4a0e3593da 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1192,7 +1192,7 @@ typedef struct { char* physicalPlan; } SMVSubscribeReq; -static FORCE_INLINE int tSerializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) { +static FORCE_INLINE int tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { int tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->topicId); tlen += taosEncodeFixedI64(buf, pReq->consumerId); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 0f600bdcf8..7aa818c7bb 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -78,41 +78,41 @@ static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { int32_t dataPos = 0; int32_t topicNum = taosArrayGetSize(pConsumer->topics); - SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId); + SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER); int32_t len = strlen(pConsumer->cgroup); SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CM_ENCODE_OVER); - SDB_SET_INT32(pRaw, dataPos, topicNum); + SDB_SET_INT32(pRaw, dataPos, topicNum, CM_ENCODE_OVER); for (int i = 0; i < topicNum; i++) { int32_t len; SMqConsumerTopic *pConsumerTopic = taosArrayGet(pConsumer->topics, i); len = strlen(pConsumerTopic->name); - SDB_SET_INT32(pRaw, dataPos, len); - SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len); + SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len, CM_ENCODE_OVER); int vgSize; if (pConsumerTopic->vgroups == NULL) { vgSize = 0; } else { vgSize = listNEles(pConsumerTopic->vgroups); } - SDB_SET_INT32(pRaw, dataPos, vgSize); + SDB_SET_INT32(pRaw, dataPos, vgSize, CM_ENCODE_OVER); for (int j = 0; j < vgSize; j++) { // SList* head; /*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/ } } - SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE); - SDB_SET_DATALEN(pRaw, dataPos); + SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER); CM_ENCODE_OVER: if (terrno != 0) { - mError("consumer:%s, failed to encode to raw:%p since %s", pConsumer->name, pRaw, terrstr()); + mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); return NULL; } - mTrace("consumer:%s, encode to raw:%p, row:%p", pConsumer->name, pRaw, pConsumer); + mTrace("consumer:%ld, encode to raw:%p, row:%p", pConsumer->consumerId, pRaw, pConsumer); return pRaw; } @@ -129,17 +129,17 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t size = sizeof(SMqConsumerObj); SSdbRow *pRow = sdbAllocRow(size); - if (pRow == NULL) goto CONSUMER_DECODE_OVER; + if (pRow == NULL) goto CONSUME_DECODE_OVER; SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); - if (pConsumer == NULL) goto CONSUMER_DECODE_OVER; + if (pConsumer == NULL) goto CONSUME_DECODE_OVER; int32_t dataPos = 0; - SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->consumerId, CONSUME_DECODE_OVER); + SDB_GET_INT64(pRaw, dataPos, &pConsumer->consumerId, CONSUME_DECODE_OVER); int32_t len, topicNum; - SDB_GET_INT32(pRaw, pRow, dataPos, &len, CONSUME_DECODE_OVER); - SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumer->cgroup, len, CONSUME_DECODE_OVER); - SDB_GET_INT32(pRaw, pRow, dataPos, &topicNum, CONSUME_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CONSUME_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER); for (int i = 0; i < topicNum; i++) { int32_t topicLen; SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); @@ -149,20 +149,21 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { return NULL; } /*pConsumerTopic->vgroups = taosArrayInit(topicNum, sizeof(SMqConsumerTopic));*/ - SDB_GET_INT32(pRaw, pRow, dataPos, &topicLen, CONSUME_DECODE_OVER); - SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumerTopic->name, topicLen, CONSUME_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &topicLen, CONSUME_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pConsumerTopic->name, topicLen, CONSUME_DECODE_OVER); int32_t vgSize; - SDB_GET_INT32(pRaw, pRow, dataPos, &vgSize, CONSUME_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER); } CONSUME_DECODE_OVER: if (terrno != 0) { - mError("consumer:%s, failed to decode from raw:%p since %s", pConsumer->name, pRaw, terrstr()); + mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); return NULL; + } - SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_CONSUMER_RESERVE_SIZE); + /*SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE);*/ return pRow; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 83870aebb7..c31b91c3fc 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -57,27 +57,26 @@ int32_t mndInitTopic(SMnode *pMnode) { void mndCleanupTopic(SMnode *pMnode) {} -static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) { - int32_t size = sizeof(STopicObj) + MND_TOPIC_RESERVE_SIZE; +SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { + int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); - if (pRaw == NULL) goto TOPIC_ENCODE_OVER; + if (pRaw == NULL) goto WTF; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN); - SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN); - SDB_SET_INT64(pRaw, dataPos, pTopic->createTime); - SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime); - SDB_SET_INT64(pRaw, dataPos, pTopic->uid); - SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid); - SDB_SET_INT32(pRaw, dataPos, pTopic->version); - SDB_SET_INT32(pRaw, dataPos, pTopic->execLen); - SDB_SET_BINARY(pRaw, dataPos, pTopic->executor, pTopic->execLen); - SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen); - SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen); + SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, WTF); + SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, WTF); + SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, WTF); + SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, WTF); + SDB_SET_INT64(pRaw, dataPos, pTopic->uid, WTF); + SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, WTF); + SDB_SET_INT32(pRaw, dataPos, pTopic->version, WTF); + SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, WTF); + SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, WTF); - SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE); - SDB_SET_DATALEN(pRaw, dataPos); + SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, WTF); + SDB_SET_DATALEN(pRaw, dataPos, WTF); +WTF: return pRaw; } @@ -109,9 +108,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); - SDB_GET_INT32(pRaw, dataPos, &len); + SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); - SDB_GET_INT32(pRaw, dataPos, &len); + SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER) @@ -374,13 +373,11 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo int32_t numOfTopics = 0; void *pIter = NULL; while (1) { - STopicObj *pTopic = NULL; + SMqTopicObj *pTopic = NULL; pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); if (pIter == NULL) break; - if (strcmp(pTopic->db, dbName) == 0) { - numOfTopics++; - } + numOfTopics++; sdbRelease(pSdb, pTopic); } From db2dbe5f519ef016245e38e31f87e18c3c0f3ac5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 20:22:33 -0800 Subject: [PATCH 33/54] pullup trans when mnode restart --- source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/src/mndSync.c | 3 +- source/dnode/mnode/impl/src/mndTrans.c | 6 +- source/dnode/mnode/impl/test/bnode/bnode.cpp | 8 +- source/dnode/mnode/impl/test/qnode/qnode.cpp | 8 +- source/dnode/mnode/impl/test/snode/snode.cpp | 8 +- source/dnode/mnode/impl/test/trans/trans.cpp | 79 ++++++++++++++++++-- 7 files changed, 87 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 93a5722842..d19718d4e0 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -64,8 +64,8 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_COMMIT_LOG = 3, - TRN_STAGE_COMMIT = 4, + TRN_STAGE_COMMIT = 3, + TRN_STAGE_COMMIT_LOG = 4, TRN_STAGE_UNDO_ACTION = 5, TRN_STAGE_UNDO_LOG = 6, TRN_STAGE_ROLLBACK = 7, diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index bf1697fb19..47d0ce4105 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -54,7 +54,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { int64_t first = walGetFirstVer(pWal); int64_t last = walGetLastVer(pWal); - mDebug("restore sdb wal start, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); + mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); first = MAX(lastSdbVer + 1, first); for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { @@ -71,6 +71,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { goto WAL_RESTORE_OVER; } + mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); if (sdbWriteNotFree(pSdb, (void *)pHead->head.body) < 0) { mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver); goto WAL_RESTORE_OVER; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 11bd15d270..4a42133ce3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -112,6 +112,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, redoLogNum, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, undoLogNum, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, commitLogNum, TRANS_ENCODE_OVER) @@ -216,6 +217,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, (int8_t *)&pTrans->policy, TRANS_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, (int8_t *)&pTrans->stage, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &redoLogNum, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &undoLogNum, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &commitLogNum, TRANS_DECODE_OVER) @@ -315,6 +317,8 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { } static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { + if (pNew->stage == TRN_STAGE_COMMIT) pNew->stage = TRN_STAGE_COMMIT_LOG; + mTrace("trans:%d, perform update action, old row:%p stage:%d, new row:%p stage:%d", pOld->id, pOld, pOld->stage, pNew, pNew->stage); pOld->stage = pNew->stage; @@ -646,7 +650,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { - mDebug("trans:%d, action:%d not sent since %s", pTrans->id, action, terrstr()); + mDebug("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } } diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index e32c0e7f91..c33776fd1b 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -186,7 +186,6 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -199,7 +198,6 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -211,7 +209,7 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateBnodeReq); @@ -250,7 +248,6 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -263,7 +260,6 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -275,7 +271,7 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateBnodeReq); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 55806dca4d..9aa41ca177 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -186,7 +186,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -199,7 +198,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -211,7 +209,7 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateQnodeReq); @@ -250,7 +248,6 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -263,7 +260,6 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -275,7 +271,7 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateQnodeReq); diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 9e07be842f..2dcfc658a4 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -186,7 +186,6 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -199,7 +198,6 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -211,7 +209,7 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateSnodeReq); @@ -250,7 +248,6 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -263,7 +260,6 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -275,7 +271,7 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateSnodeReq); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index eef94436b7..e2ed707a99 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -13,8 +13,18 @@ class MndTestTrans : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_trans", 9013); } - static void TearDownTestSuite() { test.Cleanup(); } + static void SetUpTestSuite() { + test.Init("/tmp/mnode_test_trans", 9013); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9013"; + server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; FileFd fd = taosOpenFileRead(file); @@ -40,14 +50,16 @@ class MndTestTrans : public ::testing::Test { test.ServerStart(); } - static Testbase test; + static Testbase test; + static TestServer server2; public: void SetUp() override {} void TearDown() override {} }; -Testbase MndTestTrans::test; +Testbase MndTestTrans::test; +TestServer MndTestTrans::server2; TEST_F(MndTestTrans, 01_Create_User_Crash) { { @@ -84,7 +96,7 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(MndTestTrans, 02_Create_Qnode_Crash) { +TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { { int32_t contLen = sizeof(SMCreateQnodeReq); @@ -102,9 +114,60 @@ TEST_F(MndTestTrans, 02_Create_Qnode_Crash) { } KillThenRestartServer(); + { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_QNODE_ALREADY_EXIST); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } +} + +TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { + { + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9020); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + taosMsleep(1300); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + KillThenRestartServer(); + + server2.DoStart(); + { int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateQnodeReq); @@ -118,9 +181,11 @@ TEST_F(MndTestTrans, 02_Create_Qnode_Crash) { taosMsleep(1000); } + ASSERT_NE(retry, retryMax); + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); CHECK_META("show qnodes", 3); test.SendShowRetrieveReq(); - EXPECT_EQ(test.GetShowRows(), 1); + EXPECT_EQ(test.GetShowRows(), 2); } } \ No newline at end of file From 1facebba16bb19c8fe4de38f012913a3ed12a5a0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 Jan 2022 13:15:39 +0800 Subject: [PATCH 34/54] refactor code --- source/libs/index/src/index_cache.c | 23 +++++++++++-------- .../index/src/index_fst_counting_writer.c | 1 + source/libs/index/test/fstTest.cc | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index d43c9f9cce..85e4c98276 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -27,9 +27,9 @@ static void indexMemRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl); -static void cacheTermDestroy(CacheTerm* ct); -static char* getIndexKey(const void* pData); -static int32_t compareKey(const void* l, const void* r); +static void indexCacheTermDestroy(CacheTerm* ct); +static int32_t indexCacheTermCompare(const void* l, const void* r); +static char* indexCacheTermGet(const void* pData); static MemTable* indexInternalCacheCreate(int8_t type); @@ -243,7 +243,7 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { if (mem == NULL) { return 0; } - char* key = getIndexKey(ct); + char* key = indexCacheTermGet(ct); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { @@ -321,17 +321,16 @@ void indexMemUnRef(MemTable* tbl) { } } -static void cacheTermDestroy(CacheTerm* ct) { +static void indexCacheTermDestroy(CacheTerm* ct) { if (ct == NULL) { return; } free(ct->colVal); free(ct); } -static char* getIndexKey(const void* pData) { +static char* indexCacheTermGet(const void* pData) { CacheTerm* p = (CacheTerm*)pData; return (char*)p; } - -static int32_t compareKey(const void* l, const void* r) { +static int32_t indexCacheTermCompare(const void* l, const void* r) { CacheTerm* lt = (CacheTerm*)l; CacheTerm* rt = (CacheTerm*)r; @@ -345,7 +344,8 @@ static MemTable* indexInternalCacheCreate(int8_t type) { MemTable* tbl = calloc(1, sizeof(MemTable)); indexMemRef(tbl); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); + tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, indexCacheTermCompare, SL_ALLOW_DUP_KEY, + indexCacheTermGet); } return tbl; } @@ -375,4 +375,7 @@ static bool indexCacheIteratorNext(Iterate* itera) { return next; } -static IterateValue* indexCacheIteratorGetValue(Iterate* iter) { return &iter->val; } +static IterateValue* indexCacheIteratorGetValue(Iterate* iter) { + // opt later + return &iter->val; +} diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 2b64d65e46..260c1708cb 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -125,6 +125,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { + ctx->flush(ctx); tfClose(ctx->file.fd); if (ctx->file.readOnly) { #ifdef USE_MMAP diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 3d978c05a5..70671a5f3e 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -48,7 +48,7 @@ class FstWriter { class FstReadMemory { public: - FstReadMemory(size_t size, const std::string& fileName = fileName) { + FstReadMemory(size_t size, const std::string& fileName = "/tmp/tindex.tindex") { tfInit(); _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); @@ -307,7 +307,7 @@ void validateTFile(char* arg) { tfCleanup(); } int main(int argc, char* argv[]) { - // tool to check all kind of fst test + // tool to check all kind of fst test // if (argc > 1) { validateTFile(argv[1]); } // checkFstCheckIterator(); // checkFstLongTerm(); From 41d8c7a612dafe8679c8c94df539d114dcbdae4f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 5 Jan 2022 21:20:20 -0800 Subject: [PATCH 35/54] minor changes --- source/dnode/vnode/impl/src/vnodeQuery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 5f850f0112..308fc9d2e5 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -22,12 +22,12 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vInfo("query message is processed"); + vTrace("query message is processed"); return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vInfo("fetch message is processed"); + vTrace("fetch message is processed"); switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); From ab916a479a311729c6e3ebc3f486d0bc8194eda1 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Thu, 6 Jan 2022 15:39:43 +0800 Subject: [PATCH 36/54] Revert "Feature/qnode" --- include/libs/catalog/catalog.h | 19 +-- source/libs/catalog/inc/catalogInt.h | 8 -- source/libs/catalog/src/catalog.c | 166 +++++----------------- source/libs/catalog/test/catalogTests.cpp | 7 +- source/libs/scheduler/inc/schedulerInt.h | 5 +- source/libs/scheduler/src/scheduler.c | 31 ++-- 6 files changed, 58 insertions(+), 178 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index e39236ea76..3916898829 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -50,7 +50,6 @@ typedef struct SCatalogCfg { uint32_t maxDBCacheNum; } SCatalogCfg; - int32_t catalogInit(SCatalogCfg *cfg); /** @@ -88,28 +87,15 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB */ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); -/** - * Get a super table's meta data. - * @param pCatalog (input, got with catalogGetHandle) - * @param pTransporter (input, rpc object) - * @param pMgmtEps (input, mnode EPs) - * @param pTableName (input, table name, NOT including db name) - * @param pTableMeta(output, table meta data, NEED to free it by calller) - * @return error code - */ -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); - - /** * Force renew a table's local cached meta data. * @param pCatalog (input, got with catalogGetHandle) * @param pTransporter (input, rpc object) * @param pMgmtEps (input, mnode EPs) * @param pTableName (input, table name, NOT including db name) - * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable); +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName); /** * Force renew a table's local cached meta data and get the new one. @@ -118,10 +104,9 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, co * @param pMgmtEps (input, mnode EPs) * @param pTableName (input, table name, NOT including db name) * @param pTableMeta(output, table meta data, NEED to free it by calller) - * @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure) * @return error code */ -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); /** diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c4f2b54cf8..f426139c14 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -64,14 +64,6 @@ typedef struct SCatalogMgmt { typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); -#define CTG_IS_STABLE(isSTable) (1 == (isSTable)) -#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable)) -#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0) -#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) -#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) - -#define CTG_TABLE_NOT_EXIST(code) (code == TSDB_CODE_TDB_INVALID_TABLE_ID) - #define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 9eee6f17de..045c92b586 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -105,8 +105,6 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN } *exist = 1; - - tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_SUCCESS; @@ -145,29 +143,6 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->tableCache.cache) { - return TSDB_CODE_SUCCESS; - } - - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - - size_t sz = 0; - STableMeta *pTableMeta = NULL; - - taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)&pTableMeta, &sz); - - if (NULL == pTableMeta) { - return TSDB_CODE_SUCCESS; - } - - *tbType = pTableMeta->tableType; - - return TSDB_CODE_SUCCESS; -} - - void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { epSet->inUse = 0; epSet->numOfEps = vgroupInfo->numOfEps; @@ -178,7 +153,14 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { } } -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == output) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFullName); + SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -197,12 +179,6 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pRpc, cons rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - output->metaNum = 0; - ctgDebug("tbmeta:%s not exist in mnode", tbFullName); - return TSDB_CODE_SUCCESS; - } - ctgError("error rsp for table meta, code:%x", rpcRsp.code); CTG_ERR_RET(rpcRsp.code); } @@ -212,13 +188,6 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pRpc, cons return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); - - return ctgGetTableMetaFromMnodeImpl(pCatalog, pRpc, pMgmtEps, tbFullName, output); -} - int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { @@ -228,7 +197,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE char dbFullName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFullName); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)pTableName->tname}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = pTableName->tname}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; @@ -248,12 +217,6 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { - if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - output->metaNum = 0; - ctgDebug("tbmeta:%s not exist in vnode", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - ctgError("error rsp for table meta, code:%x", rpcRsp.code); CTG_ERR_RET(rpcRsp.code); } @@ -358,28 +321,22 @@ _return: CTG_RET(TSDB_CODE_SUCCESS); } -int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { +int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; - if (!forceUpdate) { + if (!forceUpdate) { CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); - if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { + if (exist) { return TSDB_CODE_SUCCESS; } - } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { - int32_t tbType = 0; - - CTG_ERR_RET(ctgGetTableTypeFromCache(pCatalog, pTableName, &tbType)); - - CTG_SET_STABLE(isSTable, tbType); } - CTG_ERR_RET(ctgRenewTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, isSTable)); + CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pTableName)); CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); @@ -406,27 +363,19 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out } if (NULL == pCatalog->tableCache.cache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == cache) { + pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.cache, NULL, cache)) { - taosHashCleanup(cache); - } } if (NULL == pCatalog->tableCache.stableCache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == cache) { + pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->tableCache.stableCache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.stableCache, NULL, cache)) { - taosHashCleanup(cache); - } } if (output->metaNum == 2) { @@ -531,50 +480,6 @@ int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, SD return TSDB_CODE_SUCCESS; } -int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { - if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - - SVgroupInfo vgroupInfo = {0}; - int32_t code = 0; - - CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); - - STableMetaOutput voutput = {0}; - STableMetaOutput moutput = {0}; - STableMetaOutput *output = &voutput; - - if (CTG_IS_STABLE(isSTable)) { - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); - - if (0 == moutput.metaNum) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); - } else { - output = &moutput; - } - } else { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); - - if (voutput.metaNum > 0 && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); - - tfree(voutput.tbMeta); - voutput.tbMeta = moutput.tbMeta; - moutput.tbMeta = NULL; - } - } - - CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, output)); - -_return: - - tfree(voutput.tbMeta); - tfree(moutput.tbMeta); - - CTG_RET(code); -} - int32_t catalogInit(SCatalogCfg *cfg) { if (ctgMgmt.pCluster) { @@ -737,15 +642,11 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB } if (NULL == pCatalog->dbCache.cache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == cache) { + pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->dbCache.cache) { ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache.cache, NULL, cache)) { - taosHashCleanup(cache); - } } else { CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); } @@ -770,23 +671,34 @@ _return: } int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { - return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1); + return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta); } -int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { - return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1); -} - -int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - return ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable); + SVgroupInfo vgroupInfo = {0}; + int32_t code = 0; + + CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + + STableMetaOutput output = {0}; + + CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &output)); + + //CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output)); + + CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); + +_return: + tfree(output.tbMeta); + CTG_RET(code); } -int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { - return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable); +int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { + return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta); } int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 4fc53e5f18..1d8a48dfcb 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -45,7 +45,7 @@ void ctgTestSetPrepareSTableMeta(); bool ctgTestStop = false; bool ctgTestEnableSleep = false; -bool ctgTestDeadLoop = false; +bool ctgTestDeadLoop = true; int32_t ctgTestCurrentVgVersion = 0; int32_t ctgTestVgVersion = 1; @@ -600,6 +600,7 @@ void *ctgTestSetCtableMetaThread(void *param) { } +#if 0 TEST(tableMeta, normalTable) { struct SCatalog* pCtg = NULL; @@ -767,7 +768,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); tableMeta = NULL; - code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0); + code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); ASSERT_EQ(code, 0); ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); @@ -998,6 +999,8 @@ TEST(multiThread, getSetDbVgroupCase) { catalogDestroy(); } +#endif + TEST(multiThread, ctableMeta) { struct SCatalog* pCtg = NULL; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 7828eb0f5d..779a264699 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -115,9 +115,8 @@ typedef struct SSchJob { #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) -#define SCH_JOB_ELOG(param, ...) qError("QID:% "PRIx64 param, job->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:% "PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:% "PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) +#define SCH_JOB_ERR_LOG(param, ...) qError("QID:%"PRIx64 param, job->queryId, __VA_ARGS__) +#define SCH_TASK_ERR_LOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 832084154c..69df5dddee 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -280,8 +280,6 @@ int32_t schPushTaskToExecList(SSchJob *job, SSchTask *task) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("push to %s list", "execTasks"); - return TSDB_CODE_SUCCESS; } @@ -296,8 +294,6 @@ int32_t schMoveTaskToSuccList(SSchJob *job, SSchTask *task, bool *moved) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("push to %s list", "succTasks"); - *moved = true; return TSDB_CODE_SUCCESS; @@ -313,8 +309,6 @@ int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("push to %s list", "failTasks"); - *moved = true; return TSDB_CODE_SUCCESS; @@ -389,7 +383,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *job, SSchTask *task) { SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved)); if (!moved) { - SCH_TASK_ELOG("task may already moved, status:%d", task->status); + SCH_TASK_ERR_LOG(" task may already moved, status:%d", task->status); return TSDB_CODE_SUCCESS; } @@ -463,11 +457,11 @@ int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) { SCH_ERR_RET(schTaskCheckAndSetRetry(job, task, errCode, &needRetry)); if (!needRetry) { - SCH_TASK_ELOG("task failed[%x], no more retry", errCode); + SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); SCH_ERR_RET(schMoveTaskToFailList(job, task, &moved)); if (!moved) { - SCH_TASK_ELOG("task may already moved, status:%d", task->status); + SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); } if (SCH_TASK_NEED_WAIT_ALL(task)) { @@ -506,6 +500,7 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms goto _task_error; } } + break; } case TDMT_VND_SUBMIT_RSP: { @@ -582,25 +577,19 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in int32_t code = 0; SSchCallbackParam *pParam = (SSchCallbackParam *)param; - SSchJob **pjob = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); - if (NULL == pjob || NULL == (*pjob)) { + SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); + if (NULL == job || NULL == (*job)) { qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - SSchJob *job = *pjob; - - SSchTask **ptask = taosHashGet(job->execTasks, &pParam->taskId, sizeof(pParam->taskId)); - if (NULL == ptask || NULL == (*ptask)) { + SSchTask **task = taosHashGet((*job)->execTasks, &pParam->taskId, sizeof(pParam->taskId)); + if (NULL == task || NULL == (*task)) { qError("taosHashGet taskId:%"PRIx64" not exist", pParam->taskId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - - SSchTask *task = *ptask; - - SCH_TASK_DLOG("Got msg:%d, rspCode:%d", msgType, rspCode); - schProcessRspMsg(job, task, msgType, pMsg->pData, pMsg->len, rspCode); + schProcessRspMsg(*job, *task, msgType, pMsg->pData, pMsg->len, rspCode); _return: tfree(param); @@ -830,7 +819,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) { SCH_ERR_RET(schSetTaskCondidateAddrs(job, task)); if (NULL == task->condidateAddrs || taosArrayGetSize(task->condidateAddrs) <= 0) { - SCH_TASK_ELOG("no valid condidate node for task:%"PRIx64, task->taskId); + SCH_TASK_ERR_LOG("no valid condidate node for task:%"PRIx64, task->taskId); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } From 72c4f4ef16c5c19039e8ecefcda8b8f5f7269a9e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 6 Jan 2022 16:00:35 +0800 Subject: [PATCH 37/54] refactor code --- source/libs/index/inc/indexInt.h | 2 +- source/libs/index/inc/index_cache.h | 5 ++--- source/libs/index/src/index.c | 3 +-- source/libs/index/src/index_cache.c | 8 +++----- source/libs/index/src/index_fst_counting_writer.c | 2 -- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 6e1256d857..378af4c1d1 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -115,7 +115,7 @@ typedef struct TFileCacheKey { int32_t nColName; } ICacheKey; -int indexFlushCacheTFile(SIndex* sIdx, void*); +int indexFlushCacheToTFile(SIndex* sIdx, void*); int32_t indexSerialCacheKey(ICacheKey* key, char* buf); diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 544849ff2f..9db913debf 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -21,9 +21,8 @@ #include "tskiplist.h" // ----------------- key structure in skiplist --------------------- -/* A data row, the format is like below: - * content: |<--totalLen-->|<-- value len--->|<-- value -->|<--uid -->|<--version--->|<-- itermType -->| - * len : |<--int32_t -->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| +/* A data row, the format is like below + * content: |<---colVal---->|<-- version--->|<-- uid--->|<-- colType --->|<--operaType--->| */ #ifdef __cplusplus diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 0bf30d6efa..d0b8fa4290 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -384,7 +384,6 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { } } else { taosArrayPush(result, &tv); - // indexError("merge colVal: %s", tv->colVal); } } static void indexDestroyTempResult(SArray* result) { @@ -395,7 +394,7 @@ static void indexDestroyTempResult(SArray* result) { } taosArrayDestroy(result); } -int indexFlushCacheTFile(SIndex* sIdx, void* cache) { +int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { if (sIdx == NULL) { return -1; } indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 85e4c98276..8bc3776ed9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -45,9 +45,7 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in return NULL; }; cache->mem = indexInternalCacheCreate(type); - - cache->colName = calloc(1, strlen(colName) + 1); - memcpy(cache->colName, colName, strlen(colName)); + cache->colName = tstrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; @@ -187,8 +185,8 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - // pthread_mutex_unlock(&cache->mtx); pthread_cond_wait(&cache->finished, &cache->mtx); + // pthread_mutex_unlock(&cache->mtx); // taosMsleep(50); // pthread_mutex_lock(&cache->mtx); } else { @@ -353,7 +351,7 @@ static MemTable* indexInternalCacheCreate(int8_t type) { static void doMergeWork(SSchedMsg* msg) { IndexCache* pCache = msg->ahandle; SIndex* sidx = (SIndex*)pCache->index; - indexFlushCacheTFile(sidx, pCache); + indexFlushCacheToTFile(sidx, pCache); } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 260c1708cb..0763aae857 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -18,8 +18,6 @@ #include "tutil.h" static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { - // if (ctx->offset + len > ctx->limit) { return -1; } - if (ctx->type == TFile) { assert(len == tfWrite(ctx->file.fd, buf, len)); } else { From b76938130e1970b66c8e919931fbcbe9bc9dd7ab Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 6 Jan 2022 16:13:59 +0800 Subject: [PATCH 38/54] TD-12666 add vnode into vgroup one by one --- contrib/test/craft/raftServer.h | 5 +- contrib/test/traft/CMakeLists.txt | 10 +- .../test/traft/make_cluster/CMakeLists.txt | 11 + .../test/traft/{ => make_cluster}/clear.sh | 0 contrib/test/traft/make_cluster/common.h | 23 ++ contrib/test/traft/make_cluster/config.c | 64 ++++ contrib/test/traft/make_cluster/config.h | 31 ++ contrib/test/traft/make_cluster/console.c | 202 +++++++++++++ contrib/test/traft/make_cluster/console.h | 19 ++ contrib/test/traft/make_cluster/raftMain.c | 81 +++++ contrib/test/traft/make_cluster/raftServer.c | 286 ++++++++++++++++++ contrib/test/traft/make_cluster/raftServer.h | 66 ++++ contrib/test/traft/make_cluster/simpleHash.c | 218 +++++++++++++ contrib/test/traft/make_cluster/simpleHash.h | 61 ++++ contrib/test/traft/make_cluster/util.c | 45 +++ contrib/test/traft/make_cluster/util.h | 17 ++ .../traft/rebalance_leader/CMakeLists.txt | 7 + contrib/test/traft/rebalance_leader/clear.sh | 4 + .../traft/{ => rebalance_leader}/common.h | 0 .../traft/{ => rebalance_leader}/raftMain.c | 35 ++- .../traft/{ => rebalance_leader}/raftServer.c | 24 +- .../traft/{ => rebalance_leader}/raftServer.h | 12 +- 22 files changed, 1188 insertions(+), 33 deletions(-) create mode 100644 contrib/test/traft/make_cluster/CMakeLists.txt rename contrib/test/traft/{ => make_cluster}/clear.sh (100%) create mode 100644 contrib/test/traft/make_cluster/common.h create mode 100644 contrib/test/traft/make_cluster/config.c create mode 100644 contrib/test/traft/make_cluster/config.h create mode 100644 contrib/test/traft/make_cluster/console.c create mode 100644 contrib/test/traft/make_cluster/console.h create mode 100644 contrib/test/traft/make_cluster/raftMain.c create mode 100644 contrib/test/traft/make_cluster/raftServer.c create mode 100644 contrib/test/traft/make_cluster/raftServer.h create mode 100644 contrib/test/traft/make_cluster/simpleHash.c create mode 100644 contrib/test/traft/make_cluster/simpleHash.h create mode 100644 contrib/test/traft/make_cluster/util.c create mode 100644 contrib/test/traft/make_cluster/util.h create mode 100644 contrib/test/traft/rebalance_leader/CMakeLists.txt create mode 100644 contrib/test/traft/rebalance_leader/clear.sh rename contrib/test/traft/{ => rebalance_leader}/common.h (100%) rename contrib/test/traft/{ => rebalance_leader}/raftMain.c (94%) rename contrib/test/traft/{ => rebalance_leader}/raftServer.c (92%) rename contrib/test/traft/{ => rebalance_leader}/raftServer.h (93%) diff --git a/contrib/test/craft/raftServer.h b/contrib/test/craft/raftServer.h index 5fccde6bf2..f4087cf1a9 100644 --- a/contrib/test/craft/raftServer.h +++ b/contrib/test/craft/raftServer.h @@ -48,10 +48,11 @@ int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, int32_t raftServerStart(SRaftServer *pRaftServer); void raftServerClose(SRaftServer *pRaftServer); - int initFsm(struct raft_fsm *fsm); - +const char* state2String(unsigned short state); +void printRaftConfiguration(struct raft_configuration *c); +void printRaftState(struct raft *r); #ifdef __cplusplus diff --git a/contrib/test/traft/CMakeLists.txt b/contrib/test/traft/CMakeLists.txt index e29fea04f1..d165c6d4dc 100644 --- a/contrib/test/traft/CMakeLists.txt +++ b/contrib/test/traft/CMakeLists.txt @@ -1,7 +1,3 @@ -add_executable(raftMain "") -target_sources(raftMain - PRIVATE - "raftMain.c" - "raftServer.c" -) -target_link_libraries(raftMain PUBLIC traft lz4 uv_a) +add_subdirectory(rebalance_leader) +add_subdirectory(make_cluster) + diff --git a/contrib/test/traft/make_cluster/CMakeLists.txt b/contrib/test/traft/make_cluster/CMakeLists.txt new file mode 100644 index 0000000000..afd19d5435 --- /dev/null +++ b/contrib/test/traft/make_cluster/CMakeLists.txt @@ -0,0 +1,11 @@ +add_executable(makeCluster "") +target_sources(makeCluster + PRIVATE + "raftMain.c" + "raftServer.c" + "config.c" + "console.c" + "simpleHash.c" + "util.c" +) +target_link_libraries(makeCluster PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/clear.sh b/contrib/test/traft/make_cluster/clear.sh similarity index 100% rename from contrib/test/traft/clear.sh rename to contrib/test/traft/make_cluster/clear.sh diff --git a/contrib/test/traft/make_cluster/common.h b/contrib/test/traft/make_cluster/common.h new file mode 100644 index 0000000000..df7422033a --- /dev/null +++ b/contrib/test/traft/make_cluster/common.h @@ -0,0 +1,23 @@ +#ifndef TRAFT_COMMON_H +#define TRAFT_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define COMMAND_LEN 512 +#define MAX_CMD_COUNT 10 +#define TOKEN_LEN 128 +#define MAX_PEERS_COUNT 19 + +#define HOST_LEN 64 +#define ADDRESS_LEN (HOST_LEN * 2) +#define BASE_DIR_LEN 128 + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/test/traft/make_cluster/config.c b/contrib/test/traft/make_cluster/config.c new file mode 100644 index 0000000000..3b96839fd9 --- /dev/null +++ b/contrib/test/traft/make_cluster/config.c @@ -0,0 +1,64 @@ +#include "config.h" +#include +#include +#include + +void addrToString(const char *host, uint16_t port, char *addr, int len) { snprintf(addr, len, "%s:%hu", host, port); } + +void parseAddr(const char *addr, char *host, int len, uint16_t *port) { + char *tmp = (char *)malloc(strlen(addr) + 1); + strcpy(tmp, addr); + + char *context; + char *separator = ":"; + char *token = strtok_r(tmp, separator, &context); + if (token) { + snprintf(host, len, "%s", token); + } + + token = strtok_r(NULL, separator, &context); + if (token) { + sscanf(token, "%hu", port); + } + + free(tmp); +} + +int parseConf(int argc, char **argv, RaftServerConfig *pConf) { + memset(pConf, 0, sizeof(*pConf)); + + int option_index, option_value; + option_index = 0; + static struct option long_options[] = {{"help", no_argument, NULL, 'h'}, + {"addr", required_argument, NULL, 'a'}, + {"dir", required_argument, NULL, 'd'}, + {NULL, 0, NULL, 0}}; + + while ((option_value = getopt_long(argc, argv, "ha:d:", long_options, &option_index)) != -1) { + switch (option_value) { + case 'a': { + parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); + break; + } + + case 'd': { + snprintf(pConf->baseDir, sizeof(pConf->baseDir), "%s", optarg); + break; + } + + case 'h': { + return -2; + } + + default: { return -2; } + } + } + + return 0; +} + +void printConf(RaftServerConfig *pConf) { + printf("\n---printConf: \n"); + printf("me: [%s:%hu] \n", pConf->me.host, pConf->me.port); + printf("dataDir: [%s] \n\n", pConf->baseDir); +} diff --git a/contrib/test/traft/make_cluster/config.h b/contrib/test/traft/make_cluster/config.h new file mode 100644 index 0000000000..13c43d0d28 --- /dev/null +++ b/contrib/test/traft/make_cluster/config.h @@ -0,0 +1,31 @@ +#ifndef TRAFT_CONFIG_H +#define TRAFT_CONFIG_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include "common.h" + +typedef struct { + char host[HOST_LEN]; + uint16_t port; +} Addr; + +typedef struct { + Addr me; + char baseDir[BASE_DIR_LEN]; +} RaftServerConfig; + +void addrToString(const char *host, uint16_t port, char *addr, int len); +void parseAddr(const char *addr, char *host, int len, uint16_t *port); +int parseConf(int argc, char **argv, RaftServerConfig *pConf); +void printConf(RaftServerConfig *pConf); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/test/traft/make_cluster/console.c b/contrib/test/traft/make_cluster/console.c new file mode 100644 index 0000000000..b00550c681 --- /dev/null +++ b/contrib/test/traft/make_cluster/console.c @@ -0,0 +1,202 @@ +#include "console.h" +#include +#include +#include +#include "raftServer.h" +#include "util.h" + +void printHelp() { + printf("---------------------\n"); + printf("help: \n\n"); + printf("create a vgroup with 3 replicas: \n"); + printf("create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 \n"); + printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10002 \n"); + printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10001 \n"); + printf("\n"); + printf("create a vgroup with only one replica: \n"); + printf("create vnode voter vid 200 \n"); + printf("\n"); + printf("add vnode into vgroup: \n"); + printf("create vnode spare vid 100 ---- run at 127.0.0.1:10003\n"); + printf("join vnode vid 100 addr 127.0.0.1:10003 ---- run at leader of vgroup 100\n"); + printf("\n"); + printf("run \n"); + printf("put 0 key value \n"); + printf("get 0 key \n"); + printf("---------------------\n"); +} + +void console(RaftServer *pRaftServer) { + while (1) { + int ret; + char cmdBuf[COMMAND_LEN]; + memset(cmdBuf, 0, sizeof(cmdBuf)); + printf("(console)> "); + char *retp = fgets(cmdBuf, COMMAND_LEN, stdin); + if (!retp) { + exit(-1); + } + + int pos = strlen(cmdBuf); + if (cmdBuf[pos - 1] == '\n') { + cmdBuf[pos - 1] = '\0'; + } + + if (strncmp(cmdBuf, "", COMMAND_LEN) == 0) { + continue; + } + + char cmds[MAX_CMD_COUNT][TOKEN_LEN]; + memset(cmds, 0, sizeof(cmds)); + + int cmdCount; + cmdCount = splitString(cmdBuf, " ", cmds, MAX_CMD_COUNT); + + if (strcmp(cmds[0], "create") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[3], "vid") == 0) { + uint16_t vid; + sscanf(cmds[4], "%hu", &vid); + + if (strcmp(cmds[2], "voter") == 0) { + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + uint32_t peersCount = 0; + + if (strcmp(cmds[5], "peers") == 0 && cmdCount > 6) { + // create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 + for (int i = 6; i < cmdCount; ++i) { + snprintf(peers[i - 6], ADDRESS_LEN, "%s", cmds[i]); + peersCount++; + } + } else { + // create vnode voter vid 200 + } + ret = addRaftVoter(pRaftServer, peers, peersCount, vid); + if (ret == 0) { + printf("create vnode voter ok \n"); + } else { + printf("create vnode voter error \n"); + } + } else if (strcmp(cmds[2], "spare") == 0) { + ret = addRaftSpare(pRaftServer, vid); + if (ret == 0) { + printf("create vnode spare ok \n"); + } else { + printf("create vnode spare error \n"); + } + } else { + printHelp(); + } + + } else if (strcmp(cmds[0], "join") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[2], "vid") == 0 && + strcmp(cmds[4], "addr") == 0 && cmdCount == 6) { + // join vnode vid 100 addr 127.0.0.1:10004 + + char * address = cmds[5]; + char host[64]; + uint16_t port; + parseAddr(address, host, sizeof(host), &port); + + uint16_t vid; + sscanf(cmds[3], "%hu", &vid); + + HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); + if (*pp == NULL) { + printf("vid:%hu not found \n", vid); + break; + } + RaftInstance *pRaftInstance = (*pp)->data; + + uint64_t destRaftId = encodeRaftId(host, port, vid); + + struct raft_change *req = raft_malloc(sizeof(*req)); + RaftJoin * pRaftJoin = raft_malloc(sizeof(*pRaftJoin)); + pRaftJoin->r = &pRaftInstance->raft; + pRaftJoin->joinId = destRaftId; + req->data = pRaftJoin; + ret = raft_add(&pRaftInstance->raft, req, destRaftId, address, raftChangeAddCb); + if (ret != 0) { + printf("raft_add error: %s \n", raft_errmsg(&pRaftInstance->raft)); + } + + } else if (strcmp(cmds[0], "dropnode") == 0) { + } else if (strcmp(cmds[0], "state") == 0) { + pRaftServer->raftInstances.print(&pRaftServer->raftInstances); + for (size_t i = 0; i < pRaftServer->raftInstances.length; ++i) { + HashNode *ptr = pRaftServer->raftInstances.table[i]; + if (ptr != NULL) { + while (ptr != NULL) { + RaftInstance *pRaftInstance = ptr->data; + printf("instance vid:%hu raftId:%llu \n", ptr->vgroupId, pRaftInstance->raftId); + printRaftState(&pRaftInstance->raft); + printf("\n"); + ptr = ptr->next; + } + printf("\n"); + } + } + + } else if (strcmp(cmds[0], "put") == 0 && cmdCount == 4) { + uint16_t vid; + sscanf(cmds[1], "%hu", &vid); + char * key = cmds[2]; + char * value = cmds[3]; + HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); + if (*pp == NULL) { + printf("vid:%hu not found \n", vid); + break; + } + RaftInstance *pRaftInstance = (*pp)->data; + + char *raftValue = malloc(TOKEN_LEN * 2 + 3); + snprintf(raftValue, TOKEN_LEN * 2 + 3, "%s--%s", key, value); + putValue(&pRaftInstance->raft, raftValue); + free(raftValue); + + } else if (strcmp(cmds[0], "run") == 0) { + pthread_t tidRaftServer; + pthread_create(&tidRaftServer, NULL, startServerFunc, pRaftServer); + + } else if (strcmp(cmds[0], "get") == 0 && cmdCount == 3) { + uint16_t vid; + sscanf(cmds[1], "%hu", &vid); + char * key = cmds[2]; + HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); + if (*pp == NULL) { + printf("vid:%hu not found \n", vid); + break; + } + RaftInstance * pRaftInstance = (*pp)->data; + SimpleHash * pKV = pRaftInstance->fsm.data; + SimpleHashNode **ppNode = pKV->find_cstr(pKV, key); + if (*ppNode == NULL) { + printf("key:%s not found \n", key); + } else { + printf("find key:%s value:%s \n", key, (char *)((*ppNode)->data)); + } + + } else if (strcmp(cmds[0], "transfer") == 0) { + } else if (strcmp(cmds[0], "state") == 0) { + } else if (strcmp(cmds[0], "snapshot") == 0) { + } else if (strcmp(cmds[0], "exit") == 0) { + exit(0); + + } else if (strcmp(cmds[0], "quit") == 0) { + exit(0); + + } else if (strcmp(cmds[0], "help") == 0) { + printHelp(); + + } else { + printf("unknown command: %s \n", cmdBuf); + printHelp(); + } + + /* + printf("cmdBuf: [%s] \n", cmdBuf); + printf("cmdCount : %d \n", cmdCount); + for (int i = 0; i < MAX_CMD_COUNT; ++i) { + printf("cmd%d : %s \n", i, cmds[i]); + } + */ + } +} diff --git a/contrib/test/traft/make_cluster/console.h b/contrib/test/traft/make_cluster/console.h new file mode 100644 index 0000000000..f9ed12baf5 --- /dev/null +++ b/contrib/test/traft/make_cluster/console.h @@ -0,0 +1,19 @@ +#ifndef TRAFT_CONSOLE_H +#define TRAFT_CONSOLE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include "common.h" +#include "raftServer.h" + +void console(RaftServer *pRaftServer); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/test/traft/make_cluster/raftMain.c b/contrib/test/traft/make_cluster/raftMain.c new file mode 100644 index 0000000000..e25636de91 --- /dev/null +++ b/contrib/test/traft/make_cluster/raftMain.c @@ -0,0 +1,81 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "config.h" +#include "console.h" +#include "raftServer.h" +#include "simpleHash.h" +#include "util.h" + +const char *exe_name; + +void *startConsoleFunc(void *param) { + RaftServer *pRaftServer = (RaftServer *)param; + console(pRaftServer); + return NULL; +} + +void usage() { + printf("\nusage: \n"); + printf("%s --addr=127.0.0.1:10000 --dir=./data \n", exe_name); + printf("\n"); +} + +RaftServerConfig gConfig; +RaftServer gRaftServer; + +int main(int argc, char **argv) { + srand(time(NULL)); + int32_t ret; + + exe_name = argv[0]; + if (argc < 3) { + usage(); + exit(-1); + } + + ret = parseConf(argc, argv, &gConfig); + if (ret != 0) { + usage(); + exit(-1); + } + printConf(&gConfig); + + if (!dirOK(gConfig.baseDir)) { + ret = mkdir(gConfig.baseDir, 0775); + if (ret != 0) { + fprintf(stderr, "mkdir error, %s \n", gConfig.baseDir); + exit(-1); + } + } + + ret = raftServerInit(&gRaftServer, &gConfig); + if (ret != 0) { + fprintf(stderr, "raftServerInit error \n"); + exit(-1); + } + + /* + pthread_t tidRaftServer; + pthread_create(&tidRaftServer, NULL, startServerFunc, &gRaftServer); + */ + + pthread_t tidConsole; + pthread_create(&tidConsole, NULL, startConsoleFunc, &gRaftServer); + + while (1) { + sleep(10); + } + + return 0; +} diff --git a/contrib/test/traft/make_cluster/raftServer.c b/contrib/test/traft/make_cluster/raftServer.c new file mode 100644 index 0000000000..bbf67b9420 --- /dev/null +++ b/contrib/test/traft/make_cluster/raftServer.c @@ -0,0 +1,286 @@ +#include "raftServer.h" +#include +#include +#include "common.h" +#include "simpleHash.h" +#include "util.h" + +void *startServerFunc(void *param) { + RaftServer *pRaftServer = (RaftServer *)param; + int32_t r = raftServerStart(pRaftServer); + assert(r == 0); + return NULL; +} + +void raftChangeAssignCb(struct raft_change *req, int status) { + struct raft *r = req->data; + if (status != 0) { + printf("raftChangeAssignCb error: %s \n", raft_errmsg(r)); + } else { + printf("raftChangeAssignCb ok \n"); + } + raft_free(req); +} + +void raftChangeAddCb(struct raft_change *req, int status) { + RaftJoin *pRaftJoin = req->data; + if (status != 0) { + printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r)); + } else { + struct raft_change *req2 = raft_malloc(sizeof(*req2)); + req2->data = pRaftJoin->r; + int ret = raft_assign(pRaftJoin->r, req2, pRaftJoin->joinId, RAFT_VOTER, raftChangeAssignCb); + if (ret != 0) { + printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r)); + } + } + raft_free(req->data); + raft_free(req); +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { + // get fsm data + SimpleHash *sh = pFsm->data; + + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: [%s] \n", msg); + char arr[2][TOKEN_LEN]; + int r = splitString(msg, "--", arr, 2); + assert(r == 2); + + // do the value on fsm + sh->insert_cstr(sh, arr[0], arr[1]); + + raft_free(buf->base); + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + struct raft *r = req->data; + if (status != 0) { + printf("putValueCb error: %s \n", raft_errmsg(r)); + } else { + printf("putValueCb: %s \n", "ok"); + } + raft_free(req); +} + +void putValue(struct raft *r, const char *value) { + struct raft_buffer buf; + + buf.len = strlen(value) + 1; + buf.base = raft_malloc(buf.len); + snprintf(buf.base, buf.len, "%s", value); + + struct raft_apply *req = raft_malloc(sizeof(*req)); + req->data = r; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +const char *state2String(unsigned short state) { + if (state == RAFT_UNAVAILABLE) { + return "RAFT_UNAVAILABLE"; + + } else if (state == RAFT_FOLLOWER) { + return "RAFT_FOLLOWER"; + + } else if (state == RAFT_CANDIDATE) { + return "RAFT_CANDIDATE"; + + } else if (state == RAFT_LEADER) { + return "RAFT_LEADER"; + } + return "UNKNOWN_RAFT_STATE"; +} + +void printRaftConfiguration(struct raft_configuration *c) { + printf("configuration: \n"); + for (int i = 0; i < c->n; ++i) { + printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); + } +} + +void printRaftState(struct raft *r) { + printf("----Raft State: -----------\n"); + printf("mem_addr: %p \n", r); + printf("my_id: %llu \n", r->id); + printf("address: %s \n", r->address); + printf("current_term: %llu \n", r->current_term); + printf("voted_for: %llu \n", r->voted_for); + printf("role: %s \n", state2String(r->state)); + printf("commit_index: %llu \n", r->commit_index); + printf("last_applied: %llu \n", r->last_applied); + printf("last_stored: %llu \n", r->last_stored); + + printf("configuration_index: %llu \n", r->configuration_index); + printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); + printRaftConfiguration(&r->configuration); + + printf("----------------------------\n"); +} + +int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid) { + int ret; + + RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance)); + assert(pRaftInstance != NULL); + + // init raftId + pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); + + // init dir + snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir, + pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId); + + if (!dirOK(pRaftInstance->dir)) { + ret = mkdir(pRaftInstance->dir, 0775); + if (ret != 0) { + fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir); + assert(0); + } + } + + // init fsm + pRaftInstance->fsm.data = newSimpleHash(2); + pRaftInstance->fsm.apply = fsmApplyCb; + + // init io + ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport); + if (ret != 0) { + fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); + assert(0); + } + + // init raft + ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId, + pRaftServer->address); + if (ret != 0) { + fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); + assert(0); + } + + // init raft_configuration + struct raft_configuration conf; + raft_configuration_init(&conf); + raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_VOTER); + for (int i = 0; i < peersCount; ++i) { + char * peerAddress = peers[i]; + char host[64]; + uint16_t port; + parseAddr(peerAddress, host, sizeof(host), &port); + uint64_t raftId = encodeRaftId(host, port, vid); + raft_configuration_add(&conf, raftId, peers[i], RAFT_VOTER); + } + raft_bootstrap(&pRaftInstance->raft, &conf); + + // start raft + ret = raft_start(&pRaftInstance->raft); + if (ret != 0) { + fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft)); + assert(0); + } + + // add raft instance into raft server + pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance); + + return 0; +} + +int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid) { + int ret; + + RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance)); + assert(pRaftInstance != NULL); + + // init raftId + pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); + + // init dir + snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir, + pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId); + ret = mkdir(pRaftInstance->dir, 0775); + if (ret != 0) { + fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir); + assert(0); + } + + // init fsm + pRaftInstance->fsm.data = newSimpleHash(2); + pRaftInstance->fsm.apply = fsmApplyCb; + + // init io + ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport); + if (ret != 0) { + fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); + assert(0); + } + + // init raft + ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId, + pRaftServer->address); + if (ret != 0) { + fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); + assert(0); + } + + // init raft_configuration + struct raft_configuration conf; + raft_configuration_init(&conf); + raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_SPARE); + raft_bootstrap(&pRaftInstance->raft, &conf); + + // start raft + ret = raft_start(&pRaftInstance->raft); + if (ret != 0) { + fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft)); + assert(0); + } + + // add raft instance into raft server + pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance); + + return 0; +} + +int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf) { + int ret; + + // init host, port, address, dir + snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); + pRaftServer->port = pConf->me.port; + snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); + snprintf(pRaftServer->baseDir, sizeof(pRaftServer->baseDir), "%s", pConf->baseDir); + + // init loop + ret = uv_loop_init(&pRaftServer->loop); + if (ret != 0) { + fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); + assert(0); + } + + // init network + ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); + if (ret != 0) { + fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); + assert(0); + } + + // init raft instance container + initIdHash(&pRaftServer->raftInstances, 2); + + return 0; +} + +int32_t raftServerStart(RaftServer *pRaftServer) { + // start loop + uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); + return 0; +} + +void raftServerStop(RaftServer *pRaftServer) {} diff --git a/contrib/test/traft/make_cluster/raftServer.h b/contrib/test/traft/make_cluster/raftServer.h new file mode 100644 index 0000000000..b6dbddb2b7 --- /dev/null +++ b/contrib/test/traft/make_cluster/raftServer.h @@ -0,0 +1,66 @@ +#ifndef TDENGINE_RAFT_SERVER_H +#define TDENGINE_RAFT_SERVER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include "common.h" +#include "config.h" +#include "raft.h" +#include "raft/uv.h" +#include "simpleHash.h" + +typedef struct RaftJoin { + struct raft *r; + raft_id joinId; +} RaftJoin; + +typedef struct { + raft_id raftId; + char dir[BASE_DIR_LEN * 2]; + struct raft_fsm fsm; + struct raft_io io; + struct raft raft; +} RaftInstance; + +typedef struct { + char host[HOST_LEN]; + uint16_t port; + char address[ADDRESS_LEN]; /* Raft instance address */ + char baseDir[BASE_DIR_LEN]; /* Raft instance address */ + + struct uv_loop_s loop; /* UV loop */ + struct raft_uv_transport transport; /* UV I/O backend transport */ + + IdHash raftInstances; /* multi raft instances. traft use IdHash to manager multi vgroup inside, here we can use IdHash + too. */ +} RaftServer; + +void * startServerFunc(void *param); +int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid); +int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid); + +int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf); +int32_t raftServerStart(RaftServer *pRaftServer); +void raftServerStop(RaftServer *pRaftServer); + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result); +void putValueCb(struct raft_apply *req, int status, void *result); +void putValue(struct raft *r, const char *value); + +void raftChangeAddCb(struct raft_change *req, int status); + +const char *state2String(unsigned short state); +void printRaftConfiguration(struct raft_configuration *c); +void printRaftState(struct raft *r); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_RAFT_SERVER_H diff --git a/contrib/test/traft/make_cluster/simpleHash.c b/contrib/test/traft/make_cluster/simpleHash.c new file mode 100644 index 0000000000..6694843874 --- /dev/null +++ b/contrib/test/traft/make_cluster/simpleHash.c @@ -0,0 +1,218 @@ +#include "simpleHash.h" + +uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed) { + // Similar to murmur hash + const uint32_t m = 0xc6a4a793; + const uint32_t r = 24; + const char* limit = data + n; + uint32_t h = seed ^ (n * m); + + // Pick up four bytes at a time + while (data + 4 <= limit) { + // uint32_t w = DecodeFixed32(data); + uint32_t w; + memcpy(&w, data, 4); + + data += 4; + h += w; + h *= m; + h ^= (h >> 16); + } + + // Pick up remaining bytes + switch (limit - data) { + case 3: + h += (unsigned char)(data[2]) << 16; + do { + } while (0); + case 2: + h += (unsigned char)(data[1]) << 8; + do { + } while (0); + case 1: + h += (unsigned char)(data[0]); + h *= m; + h ^= (h >> r); + break; + } + return h; +} + +int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data) { + return insertSimpleHash(ths, key, strlen(key) + 1, data, strlen(data) + 1); +} + +int removeCStrSimpleHash(struct SimpleHash* ths, char* key) { return removeSimpleHash(ths, key, strlen(key) + 1); } + +SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key) { + return findSimpleHash(ths, key, strlen(key) + 1); +} + +int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen) { + SimpleHashNode** pp = ths->find(ths, key, keyLen); + if (*pp != NULL) { + fprintf(stderr, "insertSimpleHash, already has key \n"); + return -1; + } + + SimpleHashNode* node = malloc(sizeof(*node)); + node->hashCode = ths->hashFunc(key, keyLen); + node->key = malloc(keyLen); + node->keyLen = keyLen; + memcpy(node->key, key, keyLen); + node->data = malloc(dataLen); + node->dataLen = dataLen; + memcpy(node->data, data, dataLen); + node->next = NULL; + + // printf("insertSimpleHash: <%s, %ld, %s, %ld, %u> \n", node->key, node->keyLen, node->data, node->dataLen, + // node->hashCode); + + size_t index = node->hashCode & (ths->length - 1); + + SimpleHashNode* ptr = ths->table[index]; + if (ptr != NULL) { + node->next = ptr; + ths->table[index] = node; + + } else { + ths->table[index] = node; + } + ths->elems++; + if (ths->elems > 2 * ths->length) { + ths->resize(ths); + } + + return 0; +} + +int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) { + SimpleHashNode** pp = ths->find(ths, key, keyLen); + if (*pp == NULL) { + fprintf(stderr, "removeSimpleHash, key not exist \n"); + return -1; + } + + SimpleHashNode* del = *pp; + *pp = del->next; + free(del->key); + free(del->data); + free(del); + ths->elems--; + + return 0; +} + +SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) { + uint32_t hashCode = ths->hashFunc(key, keyLen); + // size_t index = hashCode % ths->length; + size_t index = hashCode & (ths->length - 1); + + // printf("findSimpleHash: %s %ld %u \n", key, keyLen, hashCode); + + SimpleHashNode** pp = &(ths->table[index]); + while (*pp != NULL && ((*pp)->hashCode != hashCode || memcmp(key, (*pp)->key, keyLen) != 0)) { + pp = &((*pp)->next); + } + + return pp; +} + +void printCStrSimpleHash(struct SimpleHash* ths) { + printf("\n--- printCStrSimpleHash: elems:%d length:%d \n", ths->elems, ths->length); + for (size_t i = 0; i < ths->length; ++i) { + SimpleHashNode* ptr = ths->table[i]; + if (ptr != NULL) { + printf("%zu: ", i); + while (ptr != NULL) { + printf("<%u, %s, %ld, %s, %ld> ", ptr->hashCode, (char*)ptr->key, ptr->keyLen, (char*)ptr->data, ptr->dataLen); + ptr = ptr->next; + } + printf("\n"); + } + } + printf("---------------\n"); +} + +void destroySimpleHash(struct SimpleHash* ths) { + for (size_t i = 0; i < ths->length; ++i) { + SimpleHashNode* ptr = ths->table[i]; + while (ptr != NULL) { + SimpleHashNode* tmp = ptr; + ptr = ptr->next; + free(tmp->key); + free(tmp->data); + free(tmp); + } + } + + ths->length = 0; + ths->elems = 0; + free(ths->table); + free(ths); +} + +void resizeSimpleHash(struct SimpleHash* ths) { + uint32_t new_length = ths->length; + while (new_length < ths->elems) { + new_length *= 2; + } + + printf("resizeSimpleHash: %p from %u to %u \n", ths, ths->length, new_length); + + SimpleHashNode** new_table = malloc(new_length * sizeof(SimpleHashNode*)); + memset(new_table, 0, new_length * sizeof(SimpleHashNode*)); + + uint32_t count = 0; + for (uint32_t i = 0; i < ths->length; i++) { + if (ths->table[i] == NULL) { + continue; + } + + SimpleHashNode* it = ths->table[i]; + while (it != NULL) { + SimpleHashNode* move_node = it; + it = it->next; + + // move move_node + move_node->next = NULL; + size_t index = move_node->hashCode & (new_length - 1); + + SimpleHashNode* ptr = new_table[index]; + if (ptr != NULL) { + move_node->next = ptr; + new_table[index] = move_node; + } else { + new_table[index] = move_node; + } + count++; + } + } + + assert(ths->elems == count); + free(ths->table); + ths->table = new_table; + ths->length = new_length; +} + +uint32_t simpleHashFunc(const char* key, size_t keyLen) { return mySimpleHash(key, keyLen, 1); } + +struct SimpleHash* newSimpleHash(size_t length) { + struct SimpleHash* ths = malloc(sizeof(*ths)); + + ths->length = length; + ths->elems = 0; + ths->table = malloc(length * sizeof(SimpleHashNode*)); + memset(ths->table, 0, length * sizeof(SimpleHashNode*)); + + ths->insert = insertSimpleHash; + ths->remove = removeSimpleHash; + ths->find = findSimpleHash; + ths->insert_cstr = insertCStrSimpleHash; + ths->remove_cstr = removeCStrSimpleHash; + ths->find_cstr = findCStrSimpleHash; + ths->print_cstr = printCStrSimpleHash; + ths->destroy = destroySimpleHash; + ths->resize = resizeSimpleHash; + ths->hashFunc = simpleHashFunc; +} diff --git a/contrib/test/traft/make_cluster/simpleHash.h b/contrib/test/traft/make_cluster/simpleHash.h new file mode 100644 index 0000000000..c6fcd93888 --- /dev/null +++ b/contrib/test/traft/make_cluster/simpleHash.h @@ -0,0 +1,61 @@ +#ifndef __SIMPLE_HASH_H__ +#define __SIMPLE_HASH_H__ + +#include +#include +#include +#include +#include + +uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed); + +typedef struct SimpleHashNode { + uint32_t hashCode; + void* key; + size_t keyLen; + void* data; + size_t dataLen; + struct SimpleHashNode* next; +} SimpleHashNode; + +typedef struct SimpleHash { + // public: + + int (*insert)(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen); + int (*remove)(struct SimpleHash* ths, char* key, size_t keyLen); + SimpleHashNode** (*find)(struct SimpleHash* ths, char* key, size_t keyLen); + + // wrapper + int (*insert_cstr)(struct SimpleHash* ths, char* key, char* data); + int (*remove_cstr)(struct SimpleHash* ths, char* key); + SimpleHashNode** (*find_cstr)(struct SimpleHash* ths, char* key); + + void (*print_cstr)(struct SimpleHash* ths); + void (*destroy)(struct SimpleHash* ths); + + uint32_t length; + uint32_t elems; + + // private: + void (*resize)(struct SimpleHash* ths); + uint32_t (*hashFunc)(const char* key, size_t keyLen); + + SimpleHashNode** table; + +} SimpleHash; + +int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data); +int removeCStrSimpleHash(struct SimpleHash* ths, char* key); +SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key); +void printCStrSimpleHash(struct SimpleHash* ths); + +int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen); +int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen); +SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen); +void destroySimpleHash(struct SimpleHash* ths); +void resizeSimpleHash(struct SimpleHash* ths); +uint32_t simpleHashFunc(const char* key, size_t keyLen); + +struct SimpleHash* newSimpleHash(size_t length); + +#endif diff --git a/contrib/test/traft/make_cluster/util.c b/contrib/test/traft/make_cluster/util.c new file mode 100644 index 0000000000..ff704f3660 --- /dev/null +++ b/contrib/test/traft/make_cluster/util.c @@ -0,0 +1,45 @@ +#include "util.h" +#include +#include +#include + +int dirOK(const char *path) { + DIR *dir = opendir(path); + if (dir != NULL) { + closedir(dir); + return 1; + } else { + return 0; + } +} + +int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr) { + if (n_arr <= 0) { + return -1; + } + + char *tmp = (char *)malloc(strlen(str) + 1); + strcpy(tmp, str); + char *context; + int n = 0; + + char *token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + strncpy(arr[n], token, TOKEN_LEN); + n++; + + while (1) { + token = strtok_r(NULL, separator, &context); + if (!token || n >= n_arr) { + goto ret; + } + strncpy(arr[n], token, TOKEN_LEN); + n++; + } + +ret: + free(tmp); + return n; +} diff --git a/contrib/test/traft/make_cluster/util.h b/contrib/test/traft/make_cluster/util.h new file mode 100644 index 0000000000..fb4ccb9c5c --- /dev/null +++ b/contrib/test/traft/make_cluster/util.h @@ -0,0 +1,17 @@ +#ifndef TRAFT_UTIL_H +#define TRAFT_UTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "common.h" + +int dirOK(const char *path); +int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/test/traft/rebalance_leader/CMakeLists.txt b/contrib/test/traft/rebalance_leader/CMakeLists.txt new file mode 100644 index 0000000000..92640bdd80 --- /dev/null +++ b/contrib/test/traft/rebalance_leader/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(rebalanceLeader "") +target_sources(rebalanceLeader + PRIVATE + "raftMain.c" + "raftServer.c" +) +target_link_libraries(rebalanceLeader PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/rebalance_leader/clear.sh b/contrib/test/traft/rebalance_leader/clear.sh new file mode 100644 index 0000000000..398b3088f2 --- /dev/null +++ b/contrib/test/traft/rebalance_leader/clear.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -rf 127.0.0.1* +rm -rf ./data diff --git a/contrib/test/traft/common.h b/contrib/test/traft/rebalance_leader/common.h similarity index 100% rename from contrib/test/traft/common.h rename to contrib/test/traft/rebalance_leader/common.h diff --git a/contrib/test/traft/raftMain.c b/contrib/test/traft/rebalance_leader/raftMain.c similarity index 94% rename from contrib/test/traft/raftMain.c rename to contrib/test/traft/rebalance_leader/raftMain.c index 24ad93856c..70dc191997 100644 --- a/contrib/test/traft/raftMain.c +++ b/contrib/test/traft/rebalance_leader/raftMain.c @@ -60,9 +60,9 @@ void raftTransferCb(struct raft_transfer *req) { SRaftServer *pRaftServer = req->data; raft_free(req); - printf("raftTransferCb: \n"); + //printf("raftTransferCb: \n"); updateLeaderStates(pRaftServer); - printLeaderCount(); + //printLeaderCount(); int myLeaderCount; for (int i = 0; i < NODE_COUNT; ++i) { @@ -71,12 +71,13 @@ void raftTransferCb(struct raft_transfer *req) { } } - printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); + //printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); if (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { struct raft *r; for (int j = 0; j < pRaftServer->instanceCount; ++j) { if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { r = &pRaftServer->instance[j].raft; + break; } } @@ -87,17 +88,25 @@ void raftTransferCb(struct raft_transfer *req) { int minIndex = -1; int minLeaderCount = myLeaderCount; for (int j = 0; j < NODE_COUNT; ++j) { - if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; + if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) { + continue; + } + if (leaderStates[j].leaderCount <= minLeaderCount) { + minLeaderCount = leaderStates[j].leaderCount; minIndex = j; } } + char myHost[48]; uint16_t myPort; uint16_t myVid; decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); - + + + //printf("raftTransferCb transfer leader: vid[%u] choose: index:%d, leaderStates[%d].address:%s, leaderStates[%d].leaderCount:%d \n", minIndex, minIndex, leaderStates[minIndex].address, minIndex, leaderStates[minIndex].leaderCount); + char *destAddress = leaderStates[minIndex].address; char tokens[MAX_PEERS][MAX_TOKEN_LEN]; @@ -106,6 +115,9 @@ void raftTransferCb(struct raft_transfer *req) { uint16_t destPort = atoi(tokens[1]); destRaftId = encodeRaftId(destHost, destPort, myVid); + printf("\nraftTransferCb transfer leader: vgroupId:%u from:%s:%u --> to:%s:%u ", myVid, myHost, myPort, destHost, destPort); + fflush(stdout); + raft_transfer(r, transfer, destRaftId, raftTransferCb); } @@ -252,7 +264,6 @@ const char* state2String(unsigned short state) { void printRaftState2(struct raft *r) { - char leaderAddress[128]; memset(leaderAddress, 0, sizeof(leaderAddress)); @@ -350,6 +361,7 @@ void console(SRaftServer *pRaftServer) { while (1) { char cmd_buf[COMMAND_LEN]; memset(cmd_buf, 0, sizeof(cmd_buf)); + printf("(console)> "); char *ret = fgets(cmd_buf, COMMAND_LEN, stdin); if (!ret) { exit(-1); @@ -403,7 +415,10 @@ void console(SRaftServer *pRaftServer) { } else if (strcmp(cmd, "dropnode") == 0) { printf("not support \n"); - } else if (strcmp(cmd, "rebalance") == 0) { + } else if (strcmp(cmd, "quit") == 0 || strcmp(cmd, "exit") == 0) { + exit(0); + + } else if (strcmp(cmd, "rebalance") == 0 && strcmp(param1, "leader") == 0) { /* updateLeaderStates(pRaftServer); @@ -511,10 +526,14 @@ void console(SRaftServer *pRaftServer) { printRaftState(&pRaftServer->instance[i].raft); } - } else if (strcmp(cmd, "state2") == 0) { + } else if (strcmp(cmd, "leader") == 0 && strcmp(param1, "state") == 0) { + updateLeaderStates(pRaftServer); + printf("\n--------------------------------------------\n"); + printLeaderCount(); for (int i = 0; i < pRaftServer->instanceCount; ++i) { printRaftState2(&pRaftServer->instance[i].raft); } + printf("--------------------------------------------\n"); } else if (strcmp(cmd, "snapshot") == 0) { printf("not support \n"); diff --git a/contrib/test/traft/raftServer.c b/contrib/test/traft/rebalance_leader/raftServer.c similarity index 92% rename from contrib/test/traft/raftServer.c rename to contrib/test/traft/rebalance_leader/raftServer.c index 94de49cd0f..165d3c9023 100644 --- a/contrib/test/traft/raftServer.c +++ b/contrib/test/traft/rebalance_leader/raftServer.c @@ -3,32 +3,34 @@ #include "common.h" #include "raftServer.h" -char *keys; -char *values; +//char *keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);; +//char *values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); + + +char keys[MAX_KV_LEN][MAX_RECORD_COUNT]; +char values[MAX_KV_LEN][MAX_RECORD_COUNT]; +int writeIndex = 0; void initStore() { - keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); - values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); - writeIndex = 0; } void destroyStore() { - free(keys); - free(values); + //free(keys); + //free(values); } void putKV(const char *key, const char *value) { if (writeIndex < MAX_RECORD_COUNT) { - strncpy(&keys[writeIndex], key, MAX_KV_LEN); - strncpy(&values[writeIndex], value, MAX_KV_LEN); + strncpy(keys[writeIndex], key, MAX_KV_LEN); + strncpy(values[writeIndex], value, MAX_KV_LEN); writeIndex++; } } char *getKV(const char *key) { for (int i = 0; i < MAX_RECORD_COUNT; ++i) { - if (strcmp(&keys[i], key) == 0) { - return &values[i]; + if (strcmp(keys[i], key) == 0) { + return values[i]; } } return NULL; diff --git a/contrib/test/traft/raftServer.h b/contrib/test/traft/rebalance_leader/raftServer.h similarity index 93% rename from contrib/test/traft/raftServer.h rename to contrib/test/traft/rebalance_leader/raftServer.h index b1f62caac5..5ea43985c9 100644 --- a/contrib/test/traft/raftServer.h +++ b/contrib/test/traft/rebalance_leader/raftServer.h @@ -15,11 +15,13 @@ extern "C" { // simulate a db store, just for test -#define MAX_KV_LEN 100 -#define MAX_RECORD_COUNT 500 -char *keys; -char *values; -int writeIndex; +#define MAX_KV_LEN 20 +#define MAX_RECORD_COUNT 16 + + +//char *keys; +//char *values; +//int writeIndex; void initStore(); void destroyStore(); From d10df1e3a892067e70c60df9fa0708013a040b69 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 00:13:49 -0800 Subject: [PATCH 39/54] minor changes --- include/common/tmsg.h | 13 +- source/client/src/clientImpl.c | 4 +- source/dnode/mgmt/impl/test/CMakeLists.txt | 1 - .../mgmt/impl/test/profile/CMakeLists.txt | 11 -- source/dnode/mnode/impl/src/mndProfile.c | 138 +++++++++--------- source/dnode/mnode/impl/src/mndShow.c | 10 +- source/dnode/mnode/impl/test/CMakeLists.txt | 1 + .../mnode/impl/test/profile/CMakeLists.txt | 11 ++ .../impl/test/profile/profile.cpp | 96 ++++++------ source/dnode/mnode/impl/test/show/show.cpp | 6 +- src/client/src/tscProfile.c | 4 +- src/client/src/tscServer.c | 10 +- tests/script/general/table/basic1.sim | 9 +- 13 files changed, 158 insertions(+), 156 deletions(-) delete mode 100644 source/dnode/mgmt/impl/test/profile/CMakeLists.txt create mode 100644 source/dnode/mnode/impl/test/profile/CMakeLists.txt rename source/dnode/{mgmt => mnode}/impl/test/profile/profile.cpp (75%) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4a0e3593da..aa36754368 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -302,7 +302,7 @@ typedef struct { char app[TSDB_APP_NAME_LEN]; char db[TSDB_DB_NAME_LEN]; int64_t startTime; -} SConnectMsg; +} SConnectReq; typedef struct SEpSet { int8_t inUse; @@ -898,7 +898,7 @@ typedef struct { int32_t numOfStreams; char app[TSDB_APP_NAME_LEN]; char pData[]; -} SHeartBeatMsg; +} SHeartBeatReq; typedef struct { int32_t connId; @@ -911,19 +911,14 @@ typedef struct { SEpSet epSet; } SHeartBeatRsp; -typedef struct { - int32_t connId; - int32_t streamId; -} SKillStreamMsg; - typedef struct { int32_t connId; int32_t queryId; -} SKillQueryMsg; +} SKillQueryReq; typedef struct { int32_t connId; -} SKillConnMsg; +} SKillConnReq; typedef struct { char user[TSDB_USER_LEN]; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 26c27a5cae..782eb7d902 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -395,13 +395,13 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) { } pMsgSendInfo->msgType = TDMT_MND_CONNECT; - pMsgSendInfo->msgInfo.len = sizeof(SConnectMsg); + pMsgSendInfo->msgInfo.len = sizeof(SConnectReq); pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->requestId = pRequest->requestId; pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; pMsgSendInfo->param = pRequest; - SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + SConnectReq *pConnect = calloc(1, sizeof(SConnectReq)); if (pConnect == NULL) { tfree(pMsgSendInfo); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index b13edf7d5a..7bab93aeb8 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -10,7 +10,6 @@ add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) -add_subdirectory(profile) add_subdirectory(stb) # add_subdirectory(sync) # add_subdirectory(telem) diff --git a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt deleted file mode 100644 index 0edd631720..0000000000 --- a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. PROFILE_SRC) -add_executable(dnode_test_profile ${PROFILE_SRC}) -target_link_libraries( - dnode_test_profile - PUBLIC sut -) - -add_test( - NAME dnode_test_profile - COMMAND dnode_test_profile -) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 77efeb8481..fcc2eec028 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -47,14 +47,14 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); -static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg); -static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg); -static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg); -static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg); -static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq); +static int32_t mndProcessConnectReq(SMnodeMsg *pReq); +static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq); +static int32_t mndProcessKillConnReq(SMnodeMsg *pReq); +static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { @@ -68,10 +68,10 @@ int32_t mndInitProfile(SMnode *pMnode) { return -1; } - mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatMsg); - mndSetMsgHandle(pMnode, TDMT_MND_CONNECT, mndProcessConnectMsg); - mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryMsg); - mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnectionMsg); + mndSetMsgHandle(pMnode, TDMT_MND_HEARTBEAT, mndProcessHeartBeatReq); + mndSetMsgHandle(pMnode, TDMT_MND_CONNECT, mndProcessConnectReq); + mndSetMsgHandle(pMnode, TDMT_MND_KILL_QUERY, mndProcessKillQueryReq); + mndSetMsgHandle(pMnode, TDMT_MND_KILL_CONN, mndProcessKillConnReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); @@ -178,35 +178,35 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); } -static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SConnectMsg *pReq = pMsg->rpcMsg.pCont; - pReq->pid = htonl(pReq->pid); - pReq->startTime = htobe64(pReq->startTime); +static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SConnectReq *pConnReq = pReq->rpcMsg.pCont; + pConnReq->pid = htonl(pConnReq->pid); + pConnReq->startTime = htobe64(pConnReq->startTime); SRpcConnInfo info = {0}; - if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { - mError("user:%s, failed to login while get connection info since %s", pMsg->user, terrstr()); + if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) { + mError("user:%s, failed to login while get connection info since %s", pReq->user, terrstr()); return -1; } char ip[30]; taosIp2String(info.clientIp, ip); - if (pReq->db[0]) { - snprintf(pMsg->db, TSDB_DB_FNAME_LEN, "%d%s%s", pMsg->acctId, TS_PATH_DELIMITER, pReq->db); - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pConnReq->db[0]) { + snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pReq->acctId, TS_PATH_DELIMITER, pConnReq->db); + SDbObj *pDb = mndAcquireDb(pMnode, pReq->db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; - mError("user:%s, failed to login from %s while use db:%s since %s", pMsg->user, ip, pReq->db, terrstr()); + mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, pConnReq->db, terrstr()); return -1; } mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, &info, pReq->pid, pReq->app, pReq->startTime); + SConnObj *pConn = mndCreateConn(pMnode, &info, pConnReq->pid, pConnReq->app, pConnReq->startTime); if (pConn == NULL) { - mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); + mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr()); return -1; } @@ -214,11 +214,11 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { if (pRsp == NULL) { mndReleaseConn(pMnode, pConn); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("user:%s, failed to login from %s while create rsp since %s", pMsg->user, ip, terrstr()); + mError("user:%s, failed to login from %s while create rsp since %s", pReq->user, ip, terrstr()); return -1; } - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); if (pUser != NULL) { pRsp->acctId = htonl(pUser->acctId); pRsp->superUser = pUser->superUser; @@ -230,16 +230,16 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndGetMnodeEpSet(pMnode, &pRsp->epSet); mndReleaseConn(pMnode, pConn); - pMsg->contLen = sizeof(SConnectRsp); - pMsg->pCont = pRsp; + pReq->contLen = sizeof(SConnectRsp); + pReq->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pReq->app); + mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pConnReq->app); return 0; } -static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { +static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { pConn->numOfQueries = 0; - int32_t numOfQueries = htonl(pMsg->numOfQueries); + int32_t numOfQueries = htonl(pReq->numOfQueries); if (numOfQueries > 0) { if (pConn->pQueries == NULL) { @@ -250,38 +250,38 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); if (saveSize > 0 && pConn->pQueries != NULL) { - memcpy(pConn->pQueries, pMsg->pData, saveSize); + memcpy(pConn->pQueries, pReq->pData, saveSize); } } return TSDB_CODE_SUCCESS; } -static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SHeartBeatMsg *pReq = pMsg->rpcMsg.pCont; - pReq->connId = htonl(pReq->connId); - pReq->pid = htonl(pReq->pid); + SHeartBeatReq *pHeartbeat = pReq->rpcMsg.pCont; + pHeartbeat->connId = htonl(pHeartbeat->connId); + pHeartbeat->pid = htonl(pHeartbeat->pid); SRpcConnInfo info = {0}; - if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { - mError("user:%s, connId:%d failed to process hb since %s", pMsg->user, pReq->connId, terrstr()); + if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) { + mError("user:%s, connId:%d failed to process hb since %s", pReq->user, pHeartbeat->connId, terrstr()); return -1; } - SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); + SConnObj *pConn = mndAcquireConn(pMnode, pHeartbeat->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, &info, pReq->pid, pReq->app, 0); + pConn = mndCreateConn(pMnode, &info, pHeartbeat->pid, pHeartbeat->app, 0); if (pConn == NULL) { - mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); + mError("user:%s, conn:%d is freed and failed to create new since %s", pReq->user, pHeartbeat->connId, terrstr()); return -1; } else { - mDebug("user:%s, conn:%d is freed and create a new conn:%d", pMsg->user, pReq->connId, pConn->id); + mDebug("user:%s, conn:%d is freed and create a new conn:%d", pReq->user, pHeartbeat->connId, pConn->id); } } else if (pConn->killed) { - mError("user:%s, conn:%d is already killed", pMsg->user, pConn->id); + mError("user:%s, conn:%d is already killed", pReq->user, pConn->id); terrno = TSDB_CODE_MND_INVALID_CONNECTION; return -1; } else { @@ -304,11 +304,11 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { if (pRsp == NULL) { mndReleaseConn(pMnode, pConn); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("user:%s, conn:%d failed to process hb while create rsp since %s", pMsg->user, pReq->connId, terrstr()); + mError("user:%s, conn:%d failed to process hb while since %s", pReq->user, pHeartbeat->connId, terrstr()); return -1; } - mndSaveQueryStreamList(pConn, pReq); + mndSaveQueryStreamList(pConn, pHeartbeat); if (pConn->killed != 0) { pRsp->killConnection = 1; } @@ -324,16 +324,16 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { mndGetMnodeEpSet(pMnode, &pRsp->epSet); mndReleaseConn(pMnode, pConn); - pMsg->contLen = sizeof(SConnectRsp); - pMsg->pCont = pRsp; + pReq->contLen = sizeof(SConnectRsp); + pReq->pCont = pRsp; return 0; } -static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) return 0; if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); @@ -342,7 +342,7 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { } mndReleaseUser(pMnode, pUser); - SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + SKillQueryReq *pKill = pReq->rpcMsg.pCont; int32_t connId = htonl(pKill->connId); int32_t queryId = htonl(pKill->queryId); mInfo("kill query msg is received, queryId:%d", pKill->queryId); @@ -353,18 +353,18 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, queryId:%d is killed by user:%s", connId, queryId, pMsg->user); + mInfo("connId:%d, queryId:%d is killed by user:%s", connId, queryId, pReq->user); pConn->queryId = queryId; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return 0; } } -static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) return 0; if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); @@ -373,7 +373,7 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { } mndReleaseUser(pMnode, pUser); - SKillConnMsg *pKill = pMsg->rpcMsg.pCont; + SKillConnReq *pKill = pReq->rpcMsg.pCont; int32_t connId = htonl(pKill->connId); SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); @@ -382,18 +382,18 @@ static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, is killed by user:%s", connId, pMsg->user); + mInfo("connId:%d, is killed by user:%s", connId, pReq->user); pConn->killed = 1; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; } } -static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) return 0; if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); @@ -464,8 +464,8 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; @@ -518,11 +518,11 @@ static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in return numOfRows; } -static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + SUserObj *pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) return 0; if (!pUser->superUser) { mndReleaseUser(pMnode, pUser); @@ -633,8 +633,8 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; int32_t numOfRows = 0; SConnObj *pConn = NULL; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 9c145f6445..28fe0551c2 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -124,20 +124,20 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show msg since %s", terrstr()); + mError("failed to process show-meta req since %s", terrstr()); return -1; } ShowMetaFp metaFp = pMgmt->metaFps[type]; if (metaFp == NULL) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); + mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); return -1; } SShowObj *pShow = mndCreateShowObj(pMnode, pShowReq); if (pShow == NULL) { - mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); + mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); return -1; } @@ -146,7 +146,7 @@ static int32_t mndProcessShowReq(SMnodeMsg *pReq) { if (pRsp == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to process show-meta msg:%s since malloc rsp error", pShow->id, + mError("show:0x%" PRIx64 ", failed to process show-meta req:%s since malloc rsp error", pShow->id, mndShowStr(type)); return -1; } @@ -181,7 +181,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { SShowObj *pShow = mndAcquireShowObj(pMnode, showId); if (pShow == NULL) { terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; - mError("failed to process show-retrieve msg:%p since %s", pShow, terrstr()); + mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 6cdc43344d..84e3ce63f9 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -7,3 +7,4 @@ add_subdirectory(qnode) add_subdirectory(snode) add_subdirectory(bnode) add_subdirectory(show) +add_subdirectory(profile) diff --git a/source/dnode/mnode/impl/test/profile/CMakeLists.txt b/source/dnode/mnode/impl/test/profile/CMakeLists.txt new file mode 100644 index 0000000000..88d7366b7d --- /dev/null +++ b/source/dnode/mnode/impl/test/profile/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. PROFILE_SRC) +add_executable(mnode_test_profile ${PROFILE_SRC}) +target_link_libraries( + mnode_test_profile + PUBLIC sut +) + +add_test( + NAME mnode_test_profile + COMMAND mnode_test_profile +) diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp similarity index 75% rename from source/dnode/mgmt/impl/test/profile/profile.cpp rename to source/dnode/mnode/impl/test/profile/profile.cpp index 29b71f1c27..bdffb6c72a 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -1,19 +1,19 @@ /** * @file profile.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module profile-msg tests - * @version 0.1 - * @date 2021-12-15 + * @brief MNODE module profile tests + * @version 1.0 + * @date 2022-01-06 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestProfile : public ::testing::Test { +class MndTestProfile : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/dnode_test_profile", 9080); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9022); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -24,15 +24,15 @@ class DndTestProfile : public ::testing::Test { void TearDown() override {} }; -Testbase DndTestProfile::test; -int32_t DndTestProfile::connId; +Testbase MndTestProfile::test; +int32_t MndTestProfile::connId; -TEST_F(DndTestProfile, 01_ConnectMsg) { - int32_t contLen = sizeof(SConnectMsg); +TEST_F(MndTestProfile, 01_ConnectMsg) { + int32_t contLen = sizeof(SConnectReq); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); + SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dnode_test_profile"); + strcpy(pReq->app, "mnode_test_profile"); strcpy(pReq->db, ""); SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); @@ -53,18 +53,18 @@ TEST_F(DndTestProfile, 01_ConnectMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9080); + EXPECT_EQ(pRsp->epSet.port[0], 9022); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; } -TEST_F(DndTestProfile, 02_ConnectMsg_InvalidDB) { - int32_t contLen = sizeof(SConnectMsg); +TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) { + int32_t contLen = sizeof(SConnectReq); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); + SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dnode_test_profile"); + strcpy(pReq->app, "mnode_test_profile"); strcpy(pReq->db, "invalid_db"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); @@ -73,7 +73,7 @@ TEST_F(DndTestProfile, 02_ConnectMsg_InvalidDB) { ASSERT_EQ(pRsp->contLen, 0); } -TEST_F(DndTestProfile, 03_ConnectMsg_Show) { +TEST_F(MndTestProfile, 03_ConnectMsg_Show) { test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); CHECK_META("show connections", 7); CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "connId"); @@ -88,22 +88,22 @@ TEST_F(DndTestProfile, 03_ConnectMsg_Show) { EXPECT_EQ(test.GetShowRows(), 1); CheckInt32(1); CheckBinary("root", TSDB_USER_LEN); - CheckBinary("dnode_test_profile", TSDB_APP_NAME_LEN); + CheckBinary("mnode_test_profile", TSDB_APP_NAME_LEN); CheckInt32(1234); IgnoreBinary(TSDB_IPv4ADDR_LEN + 6); CheckTimestamp(); CheckTimestamp(); } -TEST_F(DndTestProfile, 04_HeartBeatMsg) { - int32_t contLen = sizeof(SHeartBeatMsg); +TEST_F(MndTestProfile, 04_HeartBeatMsg) { + int32_t contLen = sizeof(SHeartBeatReq); - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); + SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dnode_test_profile"); + strcpy(pReq->app, "mnode_test_profile"); SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -127,15 +127,15 @@ TEST_F(DndTestProfile, 04_HeartBeatMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9080); + EXPECT_EQ(pRsp->epSet.port[0], 9022); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } -TEST_F(DndTestProfile, 05_KillConnMsg) { +TEST_F(MndTestProfile, 05_KillConnMsg) { { - int32_t contLen = sizeof(SKillConnMsg); + int32_t contLen = sizeof(SKillConnReq); - SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); + SKillConnReq* pReq = (SKillConnReq*)rpcMallocCont(contLen); pReq->connId = htonl(connId); SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_CONN, pReq, contLen); @@ -144,14 +144,14 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { } { - int32_t contLen = sizeof(SHeartBeatMsg); + int32_t contLen = sizeof(SHeartBeatReq); - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); + SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dnode_test_profile"); + strcpy(pReq->app, "mnode_test_profile"); SRpcMsg* pRsp = test.SendReq(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -160,11 +160,11 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { } { - int32_t contLen = sizeof(SConnectMsg); + int32_t contLen = sizeof(SConnectReq); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); + SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dnode_test_profile"); + strcpy(pReq->app, "mnode_test_profile"); strcpy(pReq->db, ""); SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); @@ -185,17 +185,17 @@ TEST_F(DndTestProfile, 05_KillConnMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9080); + EXPECT_EQ(pRsp->epSet.port[0], 9022); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; } } -TEST_F(DndTestProfile, 06_KillConnMsg_InvalidConn) { - int32_t contLen = sizeof(SKillConnMsg); +TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) { + int32_t contLen = sizeof(SKillConnReq); - SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); + SKillConnReq* pReq = (SKillConnReq*)rpcMallocCont(contLen); pReq->connId = htonl(2345); SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_CONN, pReq, contLen); @@ -203,11 +203,11 @@ TEST_F(DndTestProfile, 06_KillConnMsg_InvalidConn) { ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_CONN_ID); } -TEST_F(DndTestProfile, 07_KillQueryMsg) { +TEST_F(MndTestProfile, 07_KillQueryMsg) { { - int32_t contLen = sizeof(SKillQueryMsg); + int32_t contLen = sizeof(SKillQueryReq); - SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(contLen); + SKillQueryReq* pReq = (SKillQueryReq*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->queryId = htonl(1234); @@ -218,14 +218,14 @@ TEST_F(DndTestProfile, 07_KillQueryMsg) { } { - int32_t contLen = sizeof(SHeartBeatMsg); + int32_t contLen = sizeof(SHeartBeatReq); - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); + SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dnode_test_profile"); + strcpy(pReq->app, "mnode_test_profile"); SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -249,15 +249,15 @@ TEST_F(DndTestProfile, 07_KillQueryMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9080); + EXPECT_EQ(pRsp->epSet.port[0], 9022); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } -TEST_F(DndTestProfile, 08_KillQueryMsg_InvalidConn) { - int32_t contLen = sizeof(SKillQueryMsg); +TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) { + int32_t contLen = sizeof(SKillQueryReq); - SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(contLen); + SKillQueryReq* pReq = (SKillQueryReq*)rpcMallocCont(contLen); pReq->connId = htonl(2345); pReq->queryId = htonl(1234); @@ -266,7 +266,7 @@ TEST_F(DndTestProfile, 08_KillQueryMsg_InvalidConn) { ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_CONN_ID); } -TEST_F(DndTestProfile, 09_KillQueryMsg) { +TEST_F(MndTestProfile, 09_KillQueryMsg) { test.SendShowMetaReq(TSDB_MGMT_TABLE_QUERIES, ""); CHECK_META("show queries", 14); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index b4a625d8be..eabcbc7eb4 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -13,7 +13,7 @@ class MndTestShow : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9020); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -50,9 +50,9 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { } TEST_F(MndTestShow, 03_ShowMsg_Conn) { - int32_t contLen = sizeof(SConnectMsg); + int32_t contLen = sizeof(SConnectReq); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); + SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); pReq->pid = htonl(1234); strcpy(pReq->app, "mnode_test_show"); strcpy(pReq->db, ""); diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index e4b1602661..43e7365a59 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -226,7 +226,7 @@ void tscKillStream(STscObj *pObj, uint32_t killId) { } int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { - SHeartBeatMsg *pHeartbeat = pMsg; + SHeartBeatReq *pHeartbeat = pMsg; int allocedQueriesNum = pHeartbeat->numOfQueries; int allocedStreamsNum = pHeartbeat->numOfStreams; @@ -327,7 +327,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { } int32_t msgLen = pHeartbeat->numOfQueries * sizeof(SQueryDesc) + pHeartbeat->numOfStreams * sizeof(SStreamDesc) + - sizeof(SHeartBeatMsg); + sizeof(SHeartBeatReq); pHeartbeat->connId = htonl(pObj->connId); pHeartbeat->numOfQueries = htonl(pHeartbeat->numOfQueries); pHeartbeat->numOfStreams = htonl(pHeartbeat->numOfStreams); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e0489a76d2..a5431ea399 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1469,7 +1469,7 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SKillQueryMsg); + pCmd->payloadLen = sizeof(SKillQueryReq); switch (pCmd->command) { case TSDB_SQL_KILL_QUERY: @@ -1862,14 +1862,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STscObj *pObj = pSql->pTscObj; SSqlCmd *pCmd = &pSql->cmd; pCmd->msgType = TDMT_MND_CONNECT; - pCmd->payloadLen = sizeof(SConnectMsg); + pCmd->payloadLen = sizeof(SConnectReq); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; + SConnectReq *pConnect = (SConnectReq*)pCmd->payload; // TODO refactor full_name char *db; // ugly code to move the space @@ -1974,7 +1974,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { numOfStreams++; } - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; + int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatReq) + 100; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); @@ -1982,7 +1982,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } // TODO the expired hb and client can not be identified by server till now. - SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; + SHeartBeatReq *pHeartbeat = (SHeartBeatReq *)pCmd->payload; tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); pHeartbeat->numOfQueries = numOfQueries; diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index 298f663822..c46378beb9 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -44,13 +44,20 @@ print $data10 $data11 $data12 print =============== create child table sql create table c1 using st tags(1) -sql create table c2 using st tags(2) +sql create table c2 using st tags(2) sql show tables if $rows != 2 then return -1 endi +sql create table c3 using st tags(3) c4 using st tags(4) c5 using st tags(5) c6 using st tags(6) c7 using st tags(7) + +sql show tables +if $rows != 7 then + return -1 +endi + print $data00 $data01 $data02 print $data10 $data11 $data22 print $data20 $data11 $data22 From 283b5341dada5cc9d9a804a1ada9a75daaa3b3b0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 00:39:55 -0800 Subject: [PATCH 40/54] add batch for create_table --- tests/test/c/create_table.c | 38 +++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index 96c7f87392..c46812c9a5 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -15,25 +15,20 @@ #define _DEFAULT_SOURCE #include "os.h" - #include "taos.h" -#include "taosdef.h" #include "taoserror.h" -#include "thash.h" -#include "tutil.h" #include "ulog.h" -#define MAX_RANDOM_POINTS 20000 #define GREEN "\033[1;32m" #define NC "\033[0m" char dbName[32] = "db"; char stbName[64] = "st"; -int32_t numOfThreads = 2; +int32_t numOfThreads = 1; int32_t numOfTables = 10000; int32_t createTable = 1; int32_t insertData = 0; -int32_t batchNum = 1; +int32_t batchNum = 10; int32_t numOfVgroups = 2; typedef struct { @@ -47,11 +42,11 @@ typedef struct { pthread_t thread; } SThreadInfo; -void parseArgument(int argc, char *argv[]); +void parseArgument(int32_t argc, char *argv[]); void *threadFunc(void *param); void createDbAndStb(); -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { parseArgument(argc, argv); createDbAndStb(); @@ -64,7 +59,7 @@ int main(int argc, char *argv[]) { int32_t numOfTablesPerThread = numOfTables / numOfThreads; numOfTables = numOfTablesPerThread * numOfThreads; - for (int i = 0; i < numOfThreads; ++i) { + for (int32_t i = 0; i < numOfThreads; ++i) { pInfo[i].tableBeginIndex = i * numOfTablesPerThread; pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; pInfo[i].threadIndex = i; @@ -74,17 +69,17 @@ int main(int argc, char *argv[]) { } taosMsleep(300); - for (int i = 0; i < numOfThreads; i++) { + for (int32_t i = 0; i < numOfThreads; i++) { pthread_join(pInfo[i].thread, NULL); } float createTableSpeed = 0; - for (int i = 0; i < numOfThreads; ++i) { + for (int32_t i = 0; i < numOfThreads; ++i) { createTableSpeed += pInfo[i].createTableSpeed; } float insertDataSpeed = 0; - for (int i = 0; i < numOfThreads; ++i) { + for (int32_t i = 0; i < numOfThreads; ++i) { insertDataSpeed += pInfo[i].insertDataSpeed; } @@ -137,8 +132,8 @@ void createDbAndStb() { void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; - char qstr[65000]; - int code; + char *qstr = malloc(2000 * 1000); + int32_t code = 0; TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { @@ -153,7 +148,13 @@ void *threadFunc(void *param) { if (createTable) { int64_t startMs = taosGetTimestampMs(); for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { - sprintf(qstr, "create table t%d using %s tags(%d)", t, stbName, t); + int32_t batch = (pInfo->tableEndIndex - t); + batch = MIN(batch, batchNum); + + int32_t len = sprintf(qstr, "create table"); + for (int32_t i = 0; i < batch; ++i) { + len += sprintf(qstr + len, " t%d using %s tags(%d)", t + i, stbName, t + i); + } TAOS_RES *pSql = taos_query(con, qstr); code = taos_errno(pSql); if (code != 0) { @@ -189,6 +190,7 @@ void *threadFunc(void *param) { } taos_close(con); + free(qstr); return 0; } @@ -218,8 +220,8 @@ void printHelp() { exit(EXIT_SUCCESS); } -void parseArgument(int argc, char *argv[]) { - for (int i = 1; i < argc; i++) { +void parseArgument(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; i++) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { printHelp(); exit(0); From f7769b0966e19c4cd0ecba3d94a3e2de5bd2db5b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 00:57:08 -0800 Subject: [PATCH 41/54] minor changes --- source/dnode/mgmt/impl/test/CMakeLists.txt | 1 - .../dnode/mgmt/impl/test/dnode/CMakeLists.txt | 11 --- source/dnode/mnode/impl/test/CMakeLists.txt | 1 + .../mnode/impl/test/dnode/CMakeLists.txt | 11 +++ .../{mgmt => mnode}/impl/test/dnode/dnode.cpp | 70 +++++++++---------- 5 files changed, 47 insertions(+), 47 deletions(-) delete mode 100644 source/dnode/mgmt/impl/test/dnode/CMakeLists.txt create mode 100644 source/dnode/mnode/impl/test/dnode/CMakeLists.txt rename source/dnode/{mgmt => mnode}/impl/test/dnode/dnode.cpp (78%) diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 7bab93aeb8..26327d2c4c 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -7,7 +7,6 @@ add_subdirectory(snode) # add_subdirectory(auth) # add_subdirectory(balance) add_subdirectory(db) -add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) add_subdirectory(stb) diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt deleted file mode 100644 index 5796590865..0000000000 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -aux_source_directory(. DTEST_SRC) -add_executable(dnode_test_dnode ${DTEST_SRC}) -target_link_libraries( - dnode_test_dnode - PUBLIC sut -) - -add_test( - NAME dnode_test_dnode - COMMAND dnode_test_dnode -) diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 84e3ce63f9..ddd0869161 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -8,3 +8,4 @@ add_subdirectory(snode) add_subdirectory(bnode) add_subdirectory(show) add_subdirectory(profile) +add_subdirectory(dnode) diff --git a/source/dnode/mnode/impl/test/dnode/CMakeLists.txt b/source/dnode/mnode/impl/test/dnode/CMakeLists.txt new file mode 100644 index 0000000000..e29c5e8f3d --- /dev/null +++ b/source/dnode/mnode/impl/test/dnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. DTEST_SRC) +add_executable(mnode_test_dnode ${DTEST_SRC}) +target_link_libraries( + mnode_test_dnode + PUBLIC sut +) + +add_test( + NAME mnode_test_dnode + COMMAND mnode_test_dnode +) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mnode/impl/test/dnode/dnode.cpp similarity index 78% rename from source/dnode/mgmt/impl/test/dnode/dnode.cpp rename to source/dnode/mnode/impl/test/dnode/dnode.cpp index 51af008fc3..c5f29183e9 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/dnode.cpp @@ -1,31 +1,31 @@ /** * @file dnode.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module dnode-msg tests - * @version 0.1 - * @date 2021-12-15 + * @brief MNODE module dnode tests + * @version 1.0 + * @date 2022-01-06 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestDnode : public ::testing::Test { +class MndTestDnode : public ::testing::Test { public: void SetUp() override {} void TearDown() override {} public: static void SetUpTestSuite() { - test.Init("/tmp/dnode_test_dnode1", 9041); + test.Init("/tmp/dnode_test_dnode1", 9023); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9041"; + const char* firstEp = "localhost:9023"; - server2.Start("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); - server3.Start("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); - server4.Start("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); - server5.Start("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); + server2.Start("/tmp/dnode_test_dnode2", fqdn, 9024, firstEp); + server3.Start("/tmp/dnode_test_dnode3", fqdn, 9025, firstEp); + server4.Start("/tmp/dnode_test_dnode4", fqdn, 9026, firstEp); + server5.Start("/tmp/dnode_test_dnode5", fqdn, 9027, firstEp); taosMsleep(300); } @@ -44,13 +44,13 @@ class DndTestDnode : public ::testing::Test { static TestServer server5; }; -Testbase DndTestDnode::test; -TestServer DndTestDnode::server2; -TestServer DndTestDnode::server3; -TestServer DndTestDnode::server4; -TestServer DndTestDnode::server5; +Testbase MndTestDnode::test; +TestServer MndTestDnode::server2; +TestServer MndTestDnode::server3; +TestServer MndTestDnode::server4; +TestServer MndTestDnode::server5; -TEST_F(DndTestDnode, 01_ShowDnode) { +TEST_F(MndTestDnode, 01_ShowDnode) { test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); @@ -66,7 +66,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); - CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9023", TSDB_EP_LEN); CheckInt16(0); CheckInt16(16); CheckBinary("ready", 10); @@ -74,7 +74,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { CheckBinary("", 24); } -TEST_F(DndTestDnode, 02_ConfigDnode) { +TEST_F(MndTestDnode, 02_ConfigDnode) { int32_t contLen = sizeof(SCfgDnodeMsg); SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(contLen); @@ -86,13 +86,13 @@ TEST_F(DndTestDnode, 02_ConfigDnode) { ASSERT_EQ(pRsp->code, 0); } -TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { +TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { { int32_t contLen = sizeof(SCreateDnodeMsg); SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9042); + pReq->port = htonl(9024); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -108,8 +108,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(1); CheckInt16(2); - CheckBinary("localhost:9041", TSDB_EP_LEN); - CheckBinary("localhost:9042", TSDB_EP_LEN); + CheckBinary("localhost:9023", TSDB_EP_LEN); + CheckBinary("localhost:9024", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(16); @@ -138,7 +138,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); - CheckBinary("localhost:9041", TSDB_EP_LEN); + CheckBinary("localhost:9023", TSDB_EP_LEN); CheckInt16(0); CheckInt16(16); CheckBinary("ready", 10); @@ -150,7 +150,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9043); + pReq->port = htonl(9025); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -162,7 +162,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9044); + pReq->port = htonl(9026); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -174,7 +174,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9045); + pReq->port = htonl(9027); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -191,10 +191,10 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(3); CheckInt16(4); CheckInt16(5); - CheckBinary("localhost:9041", TSDB_EP_LEN); - CheckBinary("localhost:9043", TSDB_EP_LEN); - CheckBinary("localhost:9044", TSDB_EP_LEN); - CheckBinary("localhost:9045", TSDB_EP_LEN); + CheckBinary("localhost:9023", TSDB_EP_LEN); + CheckBinary("localhost:9025", TSDB_EP_LEN); + CheckBinary("localhost:9026", TSDB_EP_LEN); + CheckBinary("localhost:9027", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(0); @@ -234,10 +234,10 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(3); CheckInt16(4); CheckInt16(5); - CheckBinary("localhost:9041", TSDB_EP_LEN); - CheckBinary("localhost:9043", TSDB_EP_LEN); - CheckBinary("localhost:9044", TSDB_EP_LEN); - CheckBinary("localhost:9045", TSDB_EP_LEN); + CheckBinary("localhost:9023", TSDB_EP_LEN); + CheckBinary("localhost:9025", TSDB_EP_LEN); + CheckBinary("localhost:9026", TSDB_EP_LEN); + CheckBinary("localhost:9027", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); CheckInt16(0); From 93153bd2769a581834a90e676a86147f75fb38cb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 01:04:48 -0800 Subject: [PATCH 42/54] minor changes --- include/common/tmsg.h | 8 +- source/dnode/mgmt/impl/src/dndDnode.c | 8 +- source/dnode/mgmt/impl/test/mnode/mnode.cpp | 20 ++--- source/dnode/mnode/impl/src/mndDnode.c | 92 ++++++++++---------- source/dnode/mnode/impl/test/bnode/bnode.cpp | 4 +- source/dnode/mnode/impl/test/dnode/dnode.cpp | 24 ++--- source/dnode/mnode/impl/test/qnode/qnode.cpp | 4 +- source/dnode/mnode/impl/test/snode/snode.cpp | 4 +- source/dnode/mnode/impl/test/trans/trans.cpp | 4 +- source/libs/parser/inc/astToMsg.h | 4 +- source/libs/parser/src/astToMsg.c | 12 +-- source/libs/parser/src/astValidate.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/client/src/tscServer.c | 10 +-- 14 files changed, 99 insertions(+), 99 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index aa36754368..a697ecb4a7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -657,7 +657,7 @@ typedef struct { char dnodeEp[TSDB_EP_LEN]; SClusterCfg clusterCfg; SVnodeLoads vnodeLoads; -} SStatusMsg; +} SStatusReq; typedef struct { int32_t reserved; @@ -833,16 +833,16 @@ typedef struct SShowRsp { typedef struct { char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port int32_t port; -} SCreateDnodeMsg; +} SCreateDnodeReq; typedef struct { int32_t dnodeId; -} SDropDnodeMsg; +} SDropDnodeReq; typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; -} SCfgDnodeMsg; +} SMCfgDnodeReq, SDCfgDnodeReq; typedef struct { int32_t dnodeId; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index f5f9bbf1b8..5e30c4f041 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -355,9 +355,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { } void dndSendStatusReq(SDnode *pDnode) { - int32_t contLen = sizeof(SStatusMsg) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); + int32_t contLen = sizeof(SStatusReq) + TSDB_MAX_VNODES * sizeof(SVnodeLoad); - SStatusMsg *pStatus = rpcMallocCont(contLen); + SStatusReq *pStatus = rpcMallocCont(contLen); if (pStatus == NULL) { dError("failed to malloc status message"); return; @@ -385,7 +385,7 @@ void dndSendStatusReq(SDnode *pDnode) { taosRUnLockLatch(&pMgmt->latch); dndGetVnodeLoads(pDnode, &pStatus->vnodeLoads); - contLen = sizeof(SStatusMsg) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad); + contLen = sizeof(SStatusReq) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad); SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; pMgmt->statusSent = 1; @@ -464,7 +464,7 @@ static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg) { assert(1); } static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { dError("config msg is received, but not supported yet"); - SCfgDnodeMsg *pCfg = pMsg->pCont; + SDCfgDnodeReq *pCfg = pMsg->pCont; return TSDB_CODE_OPS_NOT_SUPPORT; } diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mgmt/impl/test/mnode/mnode.cpp index 9a725c20d3..468d8dc0c6 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/mnode.cpp @@ -99,9 +99,9 @@ TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { TEST_F(DndTestMnode, 04_Create_Mnode) { { // create dnode - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9062); @@ -165,9 +165,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { } } // { -// int32_t contLen = sizeof(SDropDnodeMsg); +// int32_t contLen = sizeof(SDropDnodeReq); -// SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); +// SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); // pReq->dnodeId = htonl(2); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); @@ -189,9 +189,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // CheckBinary("", 24); // { -// int32_t contLen = sizeof(SCreateDnodeMsg); +// int32_t contLen = sizeof(SCreateDnodeReq); -// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); +// SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9063"); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); @@ -200,9 +200,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // } // { -// int32_t contLen = sizeof(SCreateDnodeMsg); +// int32_t contLen = sizeof(SCreateDnodeReq); -// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); +// SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9064"); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); @@ -211,9 +211,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // } // { -// int32_t contLen = sizeof(SCreateDnodeMsg); +// int32_t contLen = sizeof(SCreateDnodeReq); -// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); +// SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); // strcpy(pReq->ep, "localhost:9065"); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2cca70b04e..91cb5ec77f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -47,17 +47,17 @@ static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode); -static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg); -static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg); -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg); -static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq); +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp); +static int32_t mndProcessStatusReq(SMnodeMsg *pReq); -static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); -static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); int32_t mndInitDnode(SMnode *pMnode) { @@ -70,11 +70,11 @@ int32_t mndInitDnode(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndDnodeActionUpdate, .deleteFp = (SdbDeleteFp)mndDnodeActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DNODE, mndProcessCreateDnodeMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_DNODE, mndProcessDropDnodeMsg); - mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DNODE, mndProcessCreateDnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_DNODE, mndProcessDropDnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); - mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusMsg); + mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndGetConfigMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndRetrieveConfigs); @@ -299,7 +299,7 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return 0; } -static void mndParseStatusMsg(SStatusMsg *pStatus) { +static void mndParseStatusMsg(SStatusReq *pStatus) { pStatus->sver = htonl(pStatus->sver); pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->clusterId = htobe64(pStatus->clusterId); @@ -311,9 +311,9 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } -static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SStatusMsg *pStatus = pMsg->rpcMsg.pCont; +static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SStatusReq *pStatus = pReq->rpcMsg.pCont; SDnodeObj *pDnode = NULL; int32_t code = -1; @@ -397,8 +397,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); - pMsg->contLen = contLen; - pMsg->pCont = pRsp; + pReq->contLen = contLen; + pReq->pCont = pRsp; } pDnode->lastAccessTime = curMs; @@ -409,7 +409,7 @@ PROCESS_STATUS_MSG_OVER: return code; } -static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { +static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pReq, SCreateDnodeReq *pCreate) { SDnodeObj dnodeObj = {0}; dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE); dnodeObj.createdTime = taosGetTimestampMs(); @@ -418,7 +418,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { mError("dnode:%s, failed to create since %s", dnodeObj.ep, terrstr()); return -1; @@ -443,9 +443,9 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * return 0; } -static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SCreateDnodeReq *pCreate = pReq->rpcMsg.pCont; pCreate->port = htonl(pCreate->port); mDebug("dnode:%s:%d, start to create", pCreate->fqdn, pCreate->port); @@ -465,7 +465,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndCreateDnode(pMnode, pMsg, pCreate); + int32_t code = mndCreateDnode(pMnode, pReq, pCreate); if (code != 0) { mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); @@ -475,8 +475,8 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); +static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pReq, SDnodeObj *pDnode) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) { mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); return -1; @@ -501,9 +501,9 @@ static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) return 0; } -static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SDropDnodeReq *pDrop = pReq->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("dnode:%d, start to drop", pDrop->dnodeId); @@ -521,7 +521,7 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropDnode(pMnode, pMsg, pDnode); + int32_t code = mndDropDnode(pMnode, pReq, pDnode); if (code != 0) { mndReleaseDnode(pMnode, pDnode); mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); @@ -532,9 +532,9 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SCfgDnodeMsg *pCfg = pMsg->rpcMsg.pCont; +static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMCfgDnodeReq *pCfg = pReq->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); @@ -547,14 +547,14 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SCfgDnodeMsg *pCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); + SDCfgDnodeReq *pCfgDnode = rpcMallocCont(sizeof(SDCfgDnodeReq)); pCfgDnode->dnodeId = htonl(pCfg->dnodeId); memcpy(pCfgDnode->config, pCfg->config, TSDB_DNODE_CONFIG_LEN); SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pCfgDnode, - .contLen = sizeof(SCfgDnodeMsg), - .ahandle = pMsg->rpcMsg.ahandle}; + .contLen = sizeof(SDCfgDnodeReq), + .ahandle = pReq->rpcMsg.ahandle}; mInfo("dnode:%d, app:%p config:%s req send to dnode", pCfg->dnodeId, rpcMsg.ahandle, pCfg->config); mndSendReqToDnode(pMnode, &epSet, &rpcMsg); @@ -562,11 +562,11 @@ static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { - mInfo("app:%p config rsp from dnode", pMsg->rpcMsg.ahandle); +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) { + mInfo("app:%p config rsp from dnode", pRsp->rpcMsg.ahandle); } -static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { +static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { int32_t cols = 0; SSchema *pSchema = pMeta->pSchema; @@ -597,8 +597,8 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg return 0; } -static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONIIG_VALUE_LEN + 1] = {0}; @@ -640,8 +640,8 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} -static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -704,8 +704,8 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * return 0; } -static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveDnodes(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; int32_t cols = 0; diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index c33776fd1b..0b54a9bf4a 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -96,9 +96,9 @@ TEST_F(MndTestBnode, 02_Create_Bnode) { TEST_F(MndTestBnode, 03_Drop_Bnode) { { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9019); diff --git a/source/dnode/mnode/impl/test/dnode/dnode.cpp b/source/dnode/mnode/impl/test/dnode/dnode.cpp index c5f29183e9..fbfc3ee7bc 100644 --- a/source/dnode/mnode/impl/test/dnode/dnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/dnode.cpp @@ -75,9 +75,9 @@ TEST_F(MndTestDnode, 01_ShowDnode) { } TEST_F(MndTestDnode, 02_ConfigDnode) { - int32_t contLen = sizeof(SCfgDnodeMsg); + int32_t contLen = sizeof(SMCfgDnodeReq); - SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(contLen); + SMCfgDnodeReq* pReq = (SMCfgDnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); @@ -88,9 +88,9 @@ TEST_F(MndTestDnode, 02_ConfigDnode) { TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9024); @@ -122,9 +122,9 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("", 24); { - int32_t contLen = sizeof(SDropDnodeMsg); + int32_t contLen = sizeof(SDropDnodeReq); - SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); + SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); @@ -146,9 +146,9 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("", 24); { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9025); @@ -158,9 +158,9 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9026); @@ -170,9 +170,9 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { } { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9027); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 9aa41ca177..8a9e087e7f 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -96,9 +96,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { TEST_F(MndTestQnode, 03_Drop_Qnode) { { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9015); diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 2dcfc658a4..3a38b9ede6 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -96,9 +96,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { TEST_F(MndTestSnode, 03_Drop_Snode) { { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9017); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index e2ed707a99..a0c4c539fb 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -133,9 +133,9 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { { - int32_t contLen = sizeof(SCreateDnodeMsg); + int32_t contLen = sizeof(SCreateDnodeReq); - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); pReq->port = htonl(9020); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 0d0769a040..1b7fe5ebc5 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -12,7 +12,7 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* ms SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); -SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); -SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); +SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); +SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 0c1af8dc3c..1ae45556b4 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -335,7 +335,7 @@ SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* p return pDropTableMsg; } -SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid host name (name too long, maximum length 128)"; const char* msg2 = "dnode name can not be string"; const char* msg3 = "port should be an integer that is less than 65535 and greater than 0"; @@ -367,7 +367,7 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs return NULL; } - SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *) calloc(1, sizeof(SCreateDnodeMsg)); + SCreateDnodeReq *pCreate = (SCreateDnodeReq *) calloc(1, sizeof(SCreateDnodeReq)); if (pCreate == NULL) { buildInvalidOperationMsg(pMsgBuf, msg4); return NULL; @@ -376,18 +376,18 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs strncpy(pCreate->fqdn, id->z, id->n); pCreate->port = htonl(val); - *len = sizeof(SCreateDnodeMsg); + *len = sizeof(SCreateDnodeReq); return pCreate; } -SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); char* end = NULL; - SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg)); + SDropDnodeReq * pDrop = (SDropDnodeReq *)calloc(1, sizeof(SDropDnodeReq)); pDrop->dnodeId = strtoll(pzName->z, &end, 10); pDrop->dnodeId = htonl(pDrop->dnodeId); - *len = sizeof(SDropDnodeMsg); + *len = sizeof(SDropDnodeReq); if (end - pzName->z != pzName->n) { buildInvalidOperationMsg(pMsgBuf, "invalid dnode id"); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index b2466fd2dc..daa960c17a 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3819,7 +3819,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer char* pMsg = pCmd->payload; - SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg; + SMCfgDnodeReq* pCfg = (SMCfgDnodeReq*)pMsg; SToken* t0 = taosArrayGet(pMiscInfo->a, 0); SToken* t1 = taosArrayGet(pMiscInfo->a, 1); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index eb3f8be25a..c68d7cef06 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -776,7 +776,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char* pMsg = pCmd->payload; - SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg; + SMCfgDnodeReq* pCfg = (SMCfgDnodeReq*)pMsg; SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0); SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index a5431ea399..41a55869e6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1192,13 +1192,13 @@ int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDnodeMsg); + pCmd->payloadLen = sizeof(SCreateDnodeReq); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; + SCreateDnodeReq *pCreate = (SCreateDnodeReq *)pCmd->payload; SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); strncpy(pCreate->ep, t0->z, t0->n); @@ -1287,7 +1287,7 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCfgDnodeMsg); + pCmd->payloadLen = sizeof(SMCfgDnodeReq); pCmd->msgType = TDMT_MND_CONFIG_DNODE; return TSDB_CODE_SUCCESS; } @@ -1350,13 +1350,13 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char dnodeEp[TSDB_EP_LEN] = {0}; tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); - pCmd->payloadLen = sizeof(SDropDnodeMsg); + pCmd->payloadLen = sizeof(SDropDnodeReq); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; + SDropDnodeReq * pDrop = (SDropDnodeReq *)pCmd->payload; tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); pCmd->msgType = TDMT_MND_DROP_DNODE; From d5e8981232ab89d745af9e2dfa0faec76d6dc658 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 02:08:13 -0800 Subject: [PATCH 43/54] Send EP list when versions are different --- include/common/tmsg.h | 2 ++ include/dnode/mnode/sdb/sdb.h | 9 ++++++ source/dnode/mgmt/impl/inc/dndInt.h | 1 + source/dnode/mgmt/impl/src/dndDnode.c | 3 ++ source/dnode/mnode/impl/src/mndDnode.c | 42 ++++++++++++++++++-------- source/dnode/mnode/sdb/src/sdbHash.c | 9 ++++++ 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a697ecb4a7..477d356818 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -650,6 +650,7 @@ typedef struct { int32_t sver; int32_t dnodeId; int64_t clusterId; + int64_t dver; int64_t rebootTime; int64_t updateTime; int32_t numOfCores; @@ -682,6 +683,7 @@ typedef struct { } SDnodeEps; typedef struct { + int64_t dver; SDnodeCfg dnodeCfg; SDnodeEps dnodeEps; } SStatusRsp; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index c7198eee6f..5a4ac6a96f 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -281,6 +281,15 @@ int32_t sdbGetSize(SSdb *pSdb, ESdbType type); */ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type); +/** + * @brief Get the version of the table + * + * @param pSdb The sdb object. + * @param pIter The type of the table. + * @return int32_t The version of the table + */ +int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type); + /** * @brief Update the version of sdb * diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 07c8ce5d02..49a688fe0d 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -83,6 +83,7 @@ typedef struct { int32_t dnodeId; int32_t dropped; int64_t clusterId; + int64_t dver; int64_t rebootTime; int64_t updateTime; int8_t statusSent; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 5e30c4f041..102e868863 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -366,6 +366,7 @@ void dndSendStatusReq(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; taosRLockLatch(&pMgmt->latch); pStatus->sver = htonl(pDnode->opt.sver); + pStatus->dver = htobe64(pMgmt->dver); pStatus->dnodeId = htonl(pMgmt->dnodeId); pStatus->clusterId = htobe64(pMgmt->clusterId); pStatus->rebootTime = htobe64(pMgmt->rebootTime); @@ -441,6 +442,8 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { SStatusRsp *pRsp = pMsg->pCont; if (pMsg->pCont != NULL && pMsg->contLen != 0) { + pMgmt->dver = htobe64(pRsp->dver); + SDnodeCfg *pCfg = &pRsp->dnodeCfg; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->clusterId = htobe64(pCfg->clusterId); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 91cb5ec77f..d94078cfe1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -45,7 +45,7 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); -static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode); +static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); static int32_t mndProcessCreateDnodeReq(SMnodeMsg *pReq); static int32_t mndProcessDropDnodeReq(SMnodeMsg *pReq); @@ -182,9 +182,9 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { return 0; } -static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { - mTrace("dnode:%d, perform update action, old_row:%p new_row:%p", pOldDnode->id, pOldDnode, pNewDnode); - pOldDnode->updateTime = pNewDnode->updateTime; +static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew) { + mTrace("dnode:%d, perform update action, old_row:%p new_row:%p", pOld->id, pOld, pNew); + pOld->updateTime = pNew->updateTime; return 0; } @@ -244,22 +244,22 @@ bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { return true; } -static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { +static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t maxEps) { SSdb *pSdb = pMnode->pSdb; - int32_t i = 0; + int32_t numOfEps = 0; void *pIter = NULL; while (1) { SDnodeObj *pDnode = NULL; pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - if (i >= numOfEps) { + if (numOfEps >= maxEps) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pDnode); break; } - SDnodeEp *pEp = &pEps->eps[i]; + SDnodeEp *pEp = &pEps->eps[numOfEps]; pEp->id = htonl(pDnode->id); pEp->port = htons(pDnode->port); memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); @@ -267,11 +267,11 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { if (mndIsMnode(pMnode, pDnode->id)) { pEp->isMnode = 1; } - i++; + numOfEps++; sdbRelease(pSdb, pDnode); } - pEps->num = htonl(i); + pEps->num = htonl(numOfEps); } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { @@ -301,6 +301,7 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { static void mndParseStatusMsg(SStatusReq *pStatus) { pStatus->sver = htonl(pStatus->sver); + pStatus->dver = htobe64(pStatus->dver); pStatus->dnodeId = htonl(pStatus->dnodeId); pStatus->clusterId = htobe64(pStatus->clusterId); pStatus->rebootTime = htobe64(pStatus->rebootTime); @@ -309,6 +310,14 @@ static void mndParseStatusMsg(SStatusReq *pStatus) { pStatus->numOfSupportVnodes = htonl(pStatus->numOfSupportVnodes); pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval); pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); + for (int32_t v = 0; v < pStatus->vnodeLoads.num; ++v) { + SVnodeLoad *pVload = &pStatus->vnodeLoads.data[v]; + pVload->vgId = htonl(pVload->vgId); + pVload->totalStorage = htobe64(pVload->totalStorage); + pVload->compStorage = htobe64(pVload->compStorage); + pVload->pointsWritten = htobe64(pVload->pointsWritten); + pVload->tablesNum = htobe64(pVload->tablesNum); + } } static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { @@ -341,9 +350,11 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { int64_t curMs = taosGetTimestampMs(); bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); - bool needCheckCfg = !(online && pDnode->rebootTime == pStatus->rebootTime); + bool dnodeChanged = (pStatus->dver != sdbGetTableVer(pMnode->pSdb, SDB_DNODE)); + bool reboot = (pDnode->rebootTime != pStatus->rebootTime); + bool needCheck = !online || dnodeChanged || reboot; - if (needCheckCfg) { + if (needCheck) { if (pStatus->sver != pMnode->cfg.sver) { if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; @@ -379,7 +390,11 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { goto PROCESS_STATUS_MSG_OVER; } - mInfo("dnode:%d, from offline to online", pDnode->id); + if (!online) { + mInfo("dnode:%d, from offline to online", pDnode->id); + } else { + mDebug("dnode:%d, send dnode eps", pDnode->id); + } pDnode->rebootTime = pStatus->rebootTime; pDnode->numOfCores = pStatus->numOfCores; @@ -393,6 +408,7 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { goto PROCESS_STATUS_MSG_OVER; } + pRsp->dver = htobe64(sdbGetTableVer(pMnode->pSdb, SDB_DNODE)); pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 13b2c7daa5..8fdb6b1657 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -429,3 +429,12 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { maxId = MAX(maxId, pSdb->maxId[type]); return maxId + 1; } + +int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type) { + if (type >= SDB_MAX || type < 0) { + terrno = TSDB_CODE_SDB_INVALID_TABLE_TYPE; + return -1; + } + + return pSdb->tableVer[type]; +} From 8a437a89dcb26288157195cf868672e88868a0c2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 6 Jan 2022 18:17:08 +0800 Subject: [PATCH 44/54] add heartbeat framework --- include/common/tmsg.h | 10 +- include/util/tcoding.h | 25 +++- include/util/thash.h | 8 ++ source/client/inc/clientHb.h | 148 ++++++++++++++++++++++ source/client/src/clientHb.c | 60 +++++++++ source/dnode/mnode/impl/src/mndConsumer.c | 33 +++-- source/dnode/mnode/impl/src/mndTopic.c | 45 ++++--- source/util/src/thash.c | 5 + 8 files changed, 300 insertions(+), 34 deletions(-) create mode 100644 source/client/inc/clientHb.h create mode 100644 source/client/src/clientHb.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4a0e3593da..7d232aa852 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1126,7 +1126,7 @@ typedef struct { int32_t topicNum; int64_t consumerId; char* consumerGroup; - char* topicName[]; + SArray* topicNames; // SArray } SCMSubscribeReq; static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -1134,8 +1134,9 @@ static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribe tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->consumerGroup); + for(int i = 0; i < pReq->topicNum; i++) { - tlen += taosEncodeString(buf, pReq->topicName[i]); + tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } return tlen; } @@ -1144,8 +1145,11 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI32(buf, &pReq->topicNum); buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeString(buf, &pReq->consumerGroup); + pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); for(int i = 0; i < pReq->topicNum; i++) { - buf = taosDecodeString(buf, &pReq->topicName[i]); + char* name = NULL; + buf = taosDecodeString(buf, &name); + taosArrayPush(pReq->topicNames, &name); } return buf; } diff --git a/include/util/tcoding.h b/include/util/tcoding.h index e1edf0d792..226856901f 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -370,10 +370,33 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { return POINTER_SHIFT(buf, size); } +// ---- binary +static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valueLen) { + int tlen = 0; + + if (buf != NULL) { + memcpy(*buf, value, valueLen); + *buf = POINTER_SHIFT(*buf, valueLen); + } + tlen += (int)valueLen; + + return tlen; +} + +static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int valueLen) { + uint64_t size = 0; + + *value = malloc((size_t)valueLen); + if (*value == NULL) return NULL; + memcpy(*value, buf, (size_t)size); + + return POINTER_SHIFT(buf, size); +} + #endif #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_CODING_H*/ \ No newline at end of file +#endif /*_TD_UTIL_CODING_H*/ diff --git a/include/util/thash.h b/include/util/thash.h index a736fc26af..4558162ac5 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -210,6 +210,14 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); */ int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); + +/** + * Get the corresponding data length for a given data in hash table + * @param data + * @return + */ +int32_t taosHashGetDataLen(void *data); + /** * return the payload data with the specified key(reference number added) * diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h new file mode 100644 index 0000000000..676b5c4c40 --- /dev/null +++ b/source/client/inc/clientHb.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tarray.h" +#include "thash.h" +#include "tmsg.h" + +typedef enum { + mq = 0, + HEARTBEAT_TYPE_MAX +} EHbType; + +typedef struct SKlv { + int32_t keyLen; + int32_t valueLen; + void* key; + void* value; +} SKlv; + +static FORCE_INLINE int taosEncodeSKlv(void** buf, const SKlv* pKlv) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKlv->keyLen); + tlen += taosEncodeFixedI32(buf, pKlv->valueLen); + tlen += taosEncodeBinary(buf, pKlv->key, pKlv->keyLen); + tlen += taosEncodeBinary(buf, pKlv->value, pKlv->valueLen); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSKlv(void* buf, SKlv* pKlv) { + buf = taosDecodeFixedI32(buf, &pKlv->keyLen); + buf = taosDecodeFixedI32(buf, &pKlv->valueLen); + buf = taosDecodeBinary(buf, &pKlv->key, pKlv->keyLen); + buf = taosDecodeBinary(buf, &pKlv->value, pKlv->valueLen); + return buf; +} + +typedef struct SClientHbKey { + int32_t connId; + int32_t hbType; +} SClientHbKey; + +static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKey->connId); + tlen += taosEncodeFixedI32(buf, pKey->hbType); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { + buf = taosDecodeFixedI32(buf, &pKey->connId); + buf = taosDecodeFixedI32(buf, &pKey->hbType); + return buf; +} + +typedef struct SClientHbReq { + SClientHbKey hbKey; + SHashObj* info; // hash +} SClientHbReq; + +static FORCE_INLINE int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq) { + int tlen = 0; + tlen += taosEncodeSClientHbKey(buf, &pReq->hbKey); + + void* pIter = NULL; + void* data; + SKlv klv; + data = taosHashIterate(pReq->info, pIter); + while (data != NULL) { + taosHashGetKey(data, &klv.key, (size_t*)&klv.keyLen); + klv.valueLen = taosHashGetDataLen(data); + klv.value = data; + taosEncodeSKlv(buf, &klv); + + data = taosHashIterate(pReq->info, pIter); + } + return tlen; +} + +static FORCE_INLINE void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq) { + ASSERT(pReq->info != NULL); + buf = taosDecodeSClientHbKey(buf, &pReq->hbKey); + + //TODO: error handling + if(pReq->info == NULL) { + pReq->info = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + SKlv klv; + buf = taosDecodeSKlv(buf, &klv); + taosHashPut(pReq->info, klv.key, klv.keyLen, klv.value, klv.valueLen); + + return buf; +} + +typedef struct SClientHbBatchReq { + int64_t reqId; + SArray* reqs; // SArray +} SClientHbBatchReq; + +typedef struct SClientHbHandleResult { +} SClientHbHandleResult; + +typedef struct SClientHbRsp { + int32_t connId; + int32_t hbType; +} SClientHbRsp; + +typedef struct SClientHbBatchRsp { + int64_t reqId; + int64_t rspId; + SArray* rsps; // SArray +} SClientHbBatchRsp; + +typedef int32_t (*FHbRspHandle)(SClientHbReq* pReq); +typedef int32_t (*FGetConnInfo)(int32_t conn, void* self); + +typedef struct SClientHbMgr { + int8_t inited; + int32_t reportInterval; // unit ms + int32_t stats; + SRWLatch lock; + SHashObj* info; //hash + FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; + // input queue +} SClientHbMgr; + +static SClientHbMgr clientHbMgr = {0}; + +int hbMgrInit(); +void hbMgrCleanUp(); + +int registerConn(int32_t connId, FGetConnInfo func, FHbRspHandle rspHandle); + +int registerHbRspHandle(int32_t connId, int32_t hbType, FHbRspHandle rspHandle); + +int HbAddConnInfo(int32_t connId, void* key, void* value, int32_t keyLen, int32_t valueLen); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c new file mode 100644 index 0000000000..f1f7409f05 --- /dev/null +++ b/source/client/src/clientHb.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "clientHb.h" + +static int32_t mqHbRspHandle(SClientHbReq* pReq) { + return 0; +} + +int hbMgrInit() { + //init once + // + //init lock + // + //init handle funcs + clientHbMgr.handle[mq] = mqHbRspHandle; + + //init stat + clientHbMgr.stats = 0; + + //init config + clientHbMgr.reportInterval = 1500; + + //init hash info + // + return 0; +} + +void hbMgrCleanUp() { + +} + +int registerConn(int32_t connId, FGetConnInfo func, FHbRspHandle rspHandle) { + return 0; +} + +int registerHbRspHandle(int32_t connId, int32_t hbType, FHbRspHandle rspHandle) { + return 0; +} + +int HbAddConnInfo(int32_t connId, void* key, void* value, int32_t keyLen, int32_t valueLen) { + //lock + + //find req by connection id + + //unlock + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7aa818c7bb..828eb4a5b7 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -127,8 +127,8 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { goto CONSUME_DECODE_OVER; } - int32_t size = sizeof(SMqConsumerObj); - SSdbRow *pRow = sdbAllocRow(size); + int32_t size = sizeof(SMqConsumerObj); + SSdbRow *pRow = sdbAllocRow(size); if (pRow == NULL) goto CONSUME_DECODE_OVER; SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); @@ -155,7 +155,6 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER); } - CONSUME_DECODE_OVER: if (terrno != 0) { mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); @@ -209,6 +208,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { tDeserializeSCMSubscribeReq(msgStr, pSubscribe); int64_t consumerId = pSubscribe->consumerId; char *consumerGroup = pSubscribe->consumerGroup; + int32_t cgroupLen = strlen(consumerGroup); SArray *newSub = NULL; int newTopicNum = pSubscribe->topicNum; @@ -216,13 +216,14 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { newSub = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); } for (int i = 0; i < newTopicNum; i++) { - char *topic = pSubscribe->topicName[i]; + char *newTopicName = taosArrayGetP(newSub, i); SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); if (pConsumerTopic == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; // TODO: free return -1; } + strcpy(pConsumerTopic->name, newTopicName); pConsumerTopic->vgroups = tdListNew(sizeof(int64_t)); taosArrayPush(newSub, pConsumerTopic); free(pConsumerTopic); @@ -239,7 +240,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - strcpy(pConsumer->cgroup, pSubscribe->consumerGroup); + pConsumer->consumerId = consumerId; + strcpy(pConsumer->cgroup, consumerGroup); } else { oldSub = pConsumer->topics; @@ -260,6 +262,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { j++; } else if (j >= oldTopicNum) { pNewTopic = taosArrayGet(newSub, i); + i++; } else { pNewTopic = taosArrayGet(newSub, i); pOldTopic = taosArrayGet(oldSub, j); @@ -292,7 +295,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName); ASSERT(pTopic != NULL); - SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup)); + SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); while ((pn = tdListNext(&iter)) != NULL) { int32_t vgId = *(int64_t *)pn->data; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); @@ -302,8 +305,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { continue; } // acquire and get epset - void *pMqVgSetReq = - mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, pSubscribe->consumerId, pSubscribe->consumerGroup); + void *pMqVgSetReq = mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, consumerId, consumerGroup); // TODO:serialize if (pMsg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -321,7 +323,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } } - taosHashRemove(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup)); + taosHashRemove(pTopic->cgroups, consumerGroup, cgroupLen); + mndReleaseTopic(pMnode, pTopic); } else if (pNewTopic != NULL) { ASSERT(pOldTopic == NULL); @@ -330,7 +333,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); ASSERT(pTopic != NULL); - SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup)); + SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); if (pGroup == NULL) { // add new group pGroup = malloc(sizeof(SMqCGroup)); @@ -346,18 +349,20 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } pGroup->status = 0; // add into cgroups - taosHashPut(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup), pGroup, - sizeof(SMqCGroup)); + taosHashPut(pTopic->cgroups, consumerGroup, cgroupLen, pGroup, sizeof(SMqCGroup)); } // put the consumer into list // rebalance will be triggered by timer - tdListAppend(pGroup->consumerIds, &pSubscribe->consumerId); + tdListAppend(pGroup->consumerIds, &consumerId); SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic); sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY); // TODO: error handling mndTransAppendRedolog(pTrans, pTopicRaw); + + mndReleaseTopic(pMnode, pTopic); + } else { ASSERT(0); } @@ -376,11 +381,13 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); + mndReleaseConsumer(pMnode, pConsumer); return -1; } // TODO: free memory mndTransDrop(pTrans); + mndReleaseConsumer(pMnode, pConsumer); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index c31b91c3fc..57b7c16c39 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -58,25 +58,36 @@ int32_t mndInitTopic(SMnode *pMnode) { void mndCleanupTopic(SMnode *pMnode) {} SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); - if (pRaw == NULL) goto WTF; + if (pRaw == NULL) goto TOPIC_ENCODE_OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, WTF); - SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, WTF); - SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, WTF); - SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, WTF); - SDB_SET_INT64(pRaw, dataPos, pTopic->uid, WTF); - SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, WTF); - SDB_SET_INT32(pRaw, dataPos, pTopic->version, WTF); - SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, WTF); - SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, WTF); + SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); + SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); + SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); - SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, WTF); - SDB_SET_DATALEN(pRaw, dataPos, WTF); + SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); + SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER); -WTF: + terrno = TSDB_CODE_SUCCESS; + +TOPIC_ENCODE_OVER: + if (terrno != TSDB_CODE_SUCCESS) { + mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("topic:%s, encode to raw:%p, row:%p", pTopic->name, pRaw, pTopic); return pRaw; } @@ -90,8 +101,8 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { goto TOPIC_DECODE_OVER; } - int32_t size = sizeof(SMqTopicObj); - SSdbRow *pRow = sdbAllocRow(size); + int32_t size = sizeof(SMqTopicObj); + SSdbRow *pRow = sdbAllocRow(size); if (pRow == NULL) goto TOPIC_DECODE_OVER; SMqTopicObj *pTopic = sdbGetRowObj(pRow); @@ -115,10 +126,10 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER) - terrno = 0; + terrno = TSDB_CODE_SUCCESS; TOPIC_DECODE_OVER: - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("topic:%s, failed to decode from raw:%p since %s", pTopic->name, pRaw, terrstr()); tfree(pRow); return NULL; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index f90b157558..181661d304 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -799,6 +799,11 @@ FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { return 0; } +FORCE_INLINE int32_t taosHashGetDataLen(void *data) { + SHashNode * node = GET_HASH_PNODE(data); + return node->keyLen; +} + FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { SHashNode * node = GET_HASH_PNODE(data); return node->keyLen; From c283a24e007f84d3d8f312de2627354a9cc6d796 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 02:25:56 -0800 Subject: [PATCH 45/54] minor changes --- source/dnode/mgmt/impl/src/dndDnode.c | 2 +- source/dnode/mgmt/impl/src/dndVnodes.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 102e868863..bd33186d82 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -350,7 +350,7 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { terrno = 0; pMgmt->updateTime = taosGetTimestampMs(); - dInfo("successed to write %s", pMgmt->file); + dDebug("successed to write %s", pMgmt->file); return 0; } diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 8835e0ba65..8a04502ae6 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -388,7 +388,7 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { free(pVnodes); } - dInfo("successed to write %s", file); + dDebug("successed to write %s", realfile); return taosRenameFile(file, realfile); } From 5839f8f6172908c2ec4e3f5e3168bae18bfec296 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 03:13:37 -0800 Subject: [PATCH 46/54] test cases --- source/dnode/mnode/impl/test/dnode/dnode.cpp | 91 +++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/test/dnode/dnode.cpp b/source/dnode/mnode/impl/test/dnode/dnode.cpp index fbfc3ee7bc..1c0cfb7bfc 100644 --- a/source/dnode/mnode/impl/test/dnode/dnode.cpp +++ b/source/dnode/mnode/impl/test/dnode/dnode.cpp @@ -86,7 +86,43 @@ TEST_F(MndTestDnode, 02_ConfigDnode) { ASSERT_EQ(pRsp->code, 0); } -TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { +TEST_F(MndTestDnode, 03_Create_Dnode) { + { + int32_t contLen = sizeof(SCreateDnodeReq); + + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, ""); + pReq->port = htonl(9024); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DNODE_EP); + } + + { + int32_t contLen = sizeof(SCreateDnodeReq); + + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(-1); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DNODE_EP); + } + + { + int32_t contLen = sizeof(SCreateDnodeReq); + + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(123456); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DNODE_EP); + } + { int32_t contLen = sizeof(SCreateDnodeReq); @@ -99,6 +135,18 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { ASSERT_EQ(pRsp->code, 0); } + { + int32_t contLen = sizeof(SCreateDnodeReq); + + SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9024); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_ALREADY_EXIST); + } + taosMsleep(1300); test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); @@ -120,6 +168,30 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckTimestamp(); CheckBinary("", 24); CheckBinary("", 24); +} + +TEST_F(MndTestDnode, 04_Drop_Dnode) { + { + int32_t contLen = sizeof(SDropDnodeReq); + + SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(-3); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_DNODE_ID); + } + + { + int32_t contLen = sizeof(SDropDnodeReq); + + SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(5); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } { int32_t contLen = sizeof(SDropDnodeReq); @@ -132,6 +204,17 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { ASSERT_EQ(pRsp->code, 0); } + { + int32_t contLen = sizeof(SDropDnodeReq); + + SDropDnodeReq* pReq = (SDropDnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); CHECK_META("show dnodes", 7); test.SendShowRetrieveReq(); @@ -145,6 +228,12 @@ TEST_F(MndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckTimestamp(); CheckBinary("", 24); + taosMsleep(2000); + server2.Stop(); + server2.DoStart(); +} + +TEST_F(MndTestDnode, 05_Create_Drop_Restart_Dnode) { { int32_t contLen = sizeof(SCreateDnodeReq); From 663ca79a5f4d0f53712d90c461f7ec2d9dddfdab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 03:18:13 -0800 Subject: [PATCH 47/54] rename files --- source/dnode/mgmt/impl/test/CMakeLists.txt | 9 +-- .../dnode/mgmt/impl/test/mnode/CMakeLists.txt | 4 +- source/dnode/mgmt/impl/test/mnode/qmnode.cpp | 26 +++++++ source/dnode/mnode/impl/test/CMakeLists.txt | 1 + .../mnode/impl/test/mnode/CMakeLists.txt | 11 +++ .../{mgmt => mnode}/impl/test/mnode/mnode.cpp | 76 +++++++++---------- 6 files changed, 79 insertions(+), 48 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/mnode/qmnode.cpp create mode 100644 source/dnode/mnode/impl/test/mnode/CMakeLists.txt rename source/dnode/{mgmt => mnode}/impl/test/mnode/mnode.cpp (79%) diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 26327d2c4c..8f721dae9a 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -3,16 +3,9 @@ enable_testing() add_subdirectory(qnode) add_subdirectory(bnode) add_subdirectory(snode) - -# add_subdirectory(auth) -# add_subdirectory(balance) -add_subdirectory(db) -# add_subdirectory(func) add_subdirectory(mnode) +add_subdirectory(db) add_subdirectory(stb) -# add_subdirectory(sync) -# add_subdirectory(telem) -# add_subdirectory(trans) add_subdirectory(vgroup) add_subdirectory(sut) diff --git a/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt index d6b3b16fb6..be29b93b02 100644 --- a/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/mnode/CMakeLists.txt @@ -1,5 +1,5 @@ -aux_source_directory(. MTEST_SRC) -add_executable(dnode_test_mnode ${MTEST_SRC}) +aux_source_directory(. DMTEST_SRC) +add_executable(dnode_test_mnode ${DMTEST_SRC}) target_link_libraries( dnode_test_mnode PUBLIC sut diff --git a/source/dnode/mgmt/impl/test/mnode/qmnode.cpp b/source/dnode/mgmt/impl/test/mnode/qmnode.cpp new file mode 100644 index 0000000000..1bf692d892 --- /dev/null +++ b/source/dnode/mgmt/impl/test/mnode/qmnode.cpp @@ -0,0 +1,26 @@ +/** + * @file dmnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module mnode tests + * @version 1.0 + * @date 2022-01-07 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class DndTestMnode : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_mnode", 9113); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} +}; + +Testbase DndTestMnode::test; diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index ddd0869161..4d37bb368d 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -9,3 +9,4 @@ add_subdirectory(bnode) add_subdirectory(show) add_subdirectory(profile) add_subdirectory(dnode) +add_subdirectory(mnode) diff --git a/source/dnode/mnode/impl/test/mnode/CMakeLists.txt b/source/dnode/mnode/impl/test/mnode/CMakeLists.txt new file mode 100644 index 0000000000..4d9b473291 --- /dev/null +++ b/source/dnode/mnode/impl/test/mnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. MTEST_SRC) +add_executable(mnode_test_mnode ${MTEST_SRC}) +target_link_libraries( + mnode_test_mnode + PUBLIC sut +) + +add_test( + NAME mnode_test_mnode + COMMAND mnode_test_mnode +) diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp similarity index 79% rename from source/dnode/mgmt/impl/test/mnode/mnode.cpp rename to source/dnode/mnode/impl/test/mnode/mnode.cpp index 468d8dc0c6..6095eb5370 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -1,31 +1,31 @@ /** - * @file dnode.cpp + * @file mnode.cpp * @author slguan (slguan@taosdata.com) - * @brief DNODE module dnode-msg tests - * @version 0.1 - * @date 2021-12-15 + * @brief MNODE module mnode tests + * @version 1.0 + * @date 2022-01-07 * - * @copyright Copyright (c) 2021 + * @copyright Copyright (c) 2022 * */ #include "sut.h" -class DndTestMnode : public ::testing::Test { +class MndTestMnode : public ::testing::Test { public: void SetUp() override {} void TearDown() override {} public: static void SetUpTestSuite() { - test.Init("/tmp/dnode_test_mnode1", 9061); + test.Init("/tmp/mnode_test_mnode1", 9031); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9061"; + const char* firstEp = "localhost:9031"; - server2.Start("/tmp/dnode_test_mnode2", fqdn, 9062, firstEp); - server3.Start("/tmp/dnode_test_mnode3", fqdn, 9063, firstEp); - server4.Start("/tmp/dnode_test_mnode4", fqdn, 9064, firstEp); - server5.Start("/tmp/dnode_test_mnode5", fqdn, 9065, firstEp); + server2.Start("/tmp/mnode_test_mnode2", fqdn, 9032, firstEp); + server3.Start("/tmp/mnode_test_mnode3", fqdn, 9033, firstEp); + server4.Start("/tmp/mnode_test_mnode4", fqdn, 9034, firstEp); + server5.Start("/tmp/mnode_test_mnode5", fqdn, 9035, firstEp); taosMsleep(300); } @@ -44,13 +44,13 @@ class DndTestMnode : public ::testing::Test { static TestServer server5; }; -Testbase DndTestMnode::test; -TestServer DndTestMnode::server2; -TestServer DndTestMnode::server3; -TestServer DndTestMnode::server4; -TestServer DndTestMnode::server5; +Testbase MndTestMnode::test; +TestServer MndTestMnode::server2; +TestServer MndTestMnode::server3; +TestServer MndTestMnode::server4; +TestServer MndTestMnode::server5; -TEST_F(DndTestMnode, 01_ShowDnode) { +TEST_F(MndTestMnode, 01_ShowDnode) { test.SendShowMetaReq(TSDB_MGMT_TABLE_MNODE, ""); CHECK_META("show mnodes", 5); @@ -64,13 +64,13 @@ TEST_F(DndTestMnode, 01_ShowDnode) { EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); - CheckBinary("localhost:9061", TSDB_EP_LEN); + CheckBinary("localhost:9031", TSDB_EP_LEN); CheckBinary("master", 12); CheckInt64(0); CheckTimestamp(); } -TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { +TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { { int32_t contLen = sizeof(SMCreateMnodeMsg); @@ -83,7 +83,7 @@ TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { } } -TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { +TEST_F(MndTestMnode, 03_Create_Mnode_Invalid_Id) { { int32_t contLen = sizeof(SMCreateMnodeMsg); @@ -96,14 +96,14 @@ TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { } } -TEST_F(DndTestMnode, 04_Create_Mnode) { +TEST_F(MndTestMnode, 04_Create_Mnode) { { // create dnode int32_t contLen = sizeof(SCreateDnodeReq); SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); strcpy(pReq->fqdn, "localhost"); - pReq->port = htonl(9062); + pReq->port = htonl(9032); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -132,8 +132,8 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { CheckInt16(1); CheckInt16(2); - CheckBinary("localhost:9061", TSDB_EP_LEN); - CheckBinary("localhost:9062", TSDB_EP_LEN); + CheckBinary("localhost:9031", TSDB_EP_LEN); + CheckBinary("localhost:9032", TSDB_EP_LEN); CheckBinary("master", 12); CheckBinary("slave", 12); CheckInt64(0); @@ -158,7 +158,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); - CheckBinary("localhost:9061", TSDB_EP_LEN); + CheckBinary("localhost:9031", TSDB_EP_LEN); CheckBinary("master", 12); CheckInt64(0); CheckTimestamp(); @@ -181,7 +181,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // EXPECT_EQ(test.GetShowRows(), 1); // CheckInt16(1); -// CheckBinary("localhost:9061", TSDB_EP_LEN); +// CheckBinary("localhost:9031", TSDB_EP_LEN); // CheckInt16(0); // CheckInt16(1); // CheckBinary("ready", 10); @@ -192,7 +192,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // int32_t contLen = sizeof(SCreateDnodeReq); // SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); -// strcpy(pReq->ep, "localhost:9063"); +// strcpy(pReq->ep, "localhost:9033"); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); // ASSERT_NE(pRsp, nullptr); @@ -203,7 +203,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // int32_t contLen = sizeof(SCreateDnodeReq); // SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); -// strcpy(pReq->ep, "localhost:9064"); +// strcpy(pReq->ep, "localhost:9034"); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); // ASSERT_NE(pRsp, nullptr); @@ -214,7 +214,7 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // int32_t contLen = sizeof(SCreateDnodeReq); // SCreateDnodeReq* pReq = (SCreateDnodeReq*)rpcMallocCont(contLen); -// strcpy(pReq->ep, "localhost:9065"); +// strcpy(pReq->ep, "localhost:9035"); // SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); // ASSERT_NE(pRsp, nullptr); @@ -231,10 +231,10 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // CheckInt16(3); // CheckInt16(4); // CheckInt16(5); -// CheckBinary("localhost:9061", TSDB_EP_LEN); -// CheckBinary("localhost:9063", TSDB_EP_LEN); -// CheckBinary("localhost:9064", TSDB_EP_LEN); -// CheckBinary("localhost:9065", TSDB_EP_LEN); +// CheckBinary("localhost:9031", TSDB_EP_LEN); +// CheckBinary("localhost:9033", TSDB_EP_LEN); +// CheckBinary("localhost:9034", TSDB_EP_LEN); +// CheckBinary("localhost:9035", TSDB_EP_LEN); // CheckInt16(0); // CheckInt16(0); // CheckInt16(0); @@ -274,10 +274,10 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { // CheckInt16(3); // CheckInt16(4); // CheckInt16(5); -// CheckBinary("localhost:9061", TSDB_EP_LEN); -// CheckBinary("localhost:9063", TSDB_EP_LEN); -// CheckBinary("localhost:9064", TSDB_EP_LEN); -// CheckBinary("localhost:9065", TSDB_EP_LEN); +// CheckBinary("localhost:9031", TSDB_EP_LEN); +// CheckBinary("localhost:9033", TSDB_EP_LEN); +// CheckBinary("localhost:9034", TSDB_EP_LEN); +// CheckBinary("localhost:9035", TSDB_EP_LEN); // CheckInt16(0); // CheckInt16(0); // CheckInt16(0); From e6a0834315fd170edaf7e8662ee1759062eaa066 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 03:36:02 -0800 Subject: [PATCH 48/54] minor changes --- source/dnode/mgmt/impl/src/dndDnode.c | 57 ++++++++++++++------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index bd33186d82..8f4287a539 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -32,10 +32,10 @@ static int32_t dndReadDnodes(SDnode *pDnode); static int32_t dndWriteDnodes(SDnode *pDnode); static void *dnodeThreadRoutine(void *param); -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg); -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg); -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg); -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg); +static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq); +static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp); +static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pRsp); +static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pRsp); int32_t dndGetDnodeId(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; @@ -80,13 +80,13 @@ void dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { taosRUnLockLatch(&pMgmt->latch); } -void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { - tmsg_t msgType = pMsg->msgType; +void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { + tmsg_t msgType = pReq->msgType; SEpSet epSet = {0}; dndGetMnodeEpSet(pDnode, &epSet); - dDebug("RPC %p, msg:%s is redirected, num:%d use:%d", pMsg->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); + dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); if (strcmp(epSet.fqdn[i], pDnode->opt.localFqdn) == 0 && epSet.port[i] == pDnode->opt.serverPort) { @@ -96,7 +96,7 @@ void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { epSet.port[i] = htons(epSet.port[i]); } - rpcSendRedirectRsp(pMsg->handle, &epSet); + rpcSendRedirectRsp(pReq->handle, &epSet); } static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -391,7 +391,7 @@ void dndSendStatusReq(SDnode *pDnode) { SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; pMgmt->statusSent = 1; - dTrace("pDnode:%p, send status msg to mnode", pDnode); + dTrace("pDnode:%p, send status req to mnode", pDnode); dndSendReqToMnode(pDnode, &rpcMsg); } @@ -427,12 +427,12 @@ static void dndUpdateDnodeEps(SDnode *pDnode, SDnodeEps *pDnodeEps) { taosWUnLockLatch(&pMgmt->latch); } -static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { +static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - if (pMsg->code != TSDB_CODE_SUCCESS) { + if (pRsp->code != TSDB_CODE_SUCCESS) { pMgmt->statusSent = 0; - if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) { + if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) { dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId); pMgmt->dropped = 1; dndWriteDnodes(pDnode); @@ -440,16 +440,16 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { return; } - SStatusRsp *pRsp = pMsg->pCont; - if (pMsg->pCont != NULL && pMsg->contLen != 0) { - pMgmt->dver = htobe64(pRsp->dver); + if (pRsp->pCont != NULL && pRsp->contLen != 0) { + SStatusRsp *pStatus = pRsp->pCont; + pMgmt->dver = htobe64(pStatus->dver); - SDnodeCfg *pCfg = &pRsp->dnodeCfg; + SDnodeCfg *pCfg = &pStatus->dnodeCfg; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->clusterId = htobe64(pCfg->clusterId); dndUpdateDnodeCfg(pDnode, pCfg); - SDnodeEps *pDnodeEps = &pRsp->dnodeEps; + SDnodeEps *pDnodeEps = &pStatus->dnodeEps; pDnodeEps->num = htonl(pDnodeEps->num); for (int32_t i = 0; i < pDnodeEps->num; ++i) { pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); @@ -461,26 +461,27 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { pMgmt->statusSent = 0; } -static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pMsg) { assert(1); } +static void dndProcessAuthRsp(SDnode *pDnode, SRpcMsg *pReq) { dError("auth rsp is received, but not supported yet"); } -static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pMsg) { assert(1); } - -static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pMsg) { - dError("config msg is received, but not supported yet"); - SDCfgDnodeReq *pCfg = pMsg->pCont; +static void dndProcessGrantRsp(SDnode *pDnode, SRpcMsg *pReq) { + dError("grant rsp is received, but not supported yet"); +} +static int32_t dndProcessConfigDnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + dError("config req is received, but not supported yet"); + SDCfgDnodeReq *pCfg = pReq->pCont; return TSDB_CODE_OPS_NOT_SUPPORT; } -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg) { - dDebug("startup msg is received"); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); SStartupMsg *pStartup = rpcMallocCont(sizeof(SStartupMsg)); dndGetStartup(pDnode, pStartup); - dDebug("startup msg is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = {.handle = pMsg->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)}; + SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupMsg)}; rpcSendResponse(&rpcRsp); } @@ -707,7 +708,7 @@ static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; code = -1; - dError("RPC %p, dnode req:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); + dError("RPC %p, dnode msg:%s not processed", pMsg->handle, TMSG_INFO(pMsg->msgType)); break; } From f34013a06eba750b0951f89c15f7c450353feafa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 03:40:20 -0800 Subject: [PATCH 49/54] fix invalid free --- source/dnode/mgmt/impl/src/dndWorker.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c index b1107fd185..e0db262f89 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -101,7 +101,9 @@ int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) } if (taosWriteQitem(pWorker->queue, pMsg) != 0) { - taosFreeQitem(pMsg); + if (contLen != 0) { + taosFreeQitem(pMsg); + } terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } From 3af4fe31a58b9dacdc4f44b2969e510514a318a9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 03:50:15 -0800 Subject: [PATCH 50/54] refact dnode mgmt worker --- source/dnode/mgmt/impl/inc/dndInt.h | 29 +++++------ source/dnode/mgmt/impl/src/dndDnode.c | 75 +++++---------------------- 2 files changed, 27 insertions(+), 77 deletions(-) diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 49a688fe0d..d5c9dd57dc 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -80,21 +80,20 @@ typedef struct { } SDnodeDir; typedef struct { - int32_t dnodeId; - int32_t dropped; - int64_t clusterId; - int64_t dver; - int64_t rebootTime; - int64_t updateTime; - int8_t statusSent; - SEpSet mnodeEpSet; - char *file; - SHashObj *dnodeHash; - SDnodeEps *dnodeEps; - pthread_t *threadId; - SRWLatch latch; - STaosQueue *pMgmtQ; - SWorkerPool mgmtPool; + int32_t dnodeId; + int32_t dropped; + int64_t clusterId; + int64_t dver; + int64_t rebootTime; + int64_t updateTime; + int8_t statusSent; + SEpSet mnodeEpSet; + char *file; + SHashObj *dnodeHash; + SDnodeEps *dnodeEps; + pthread_t *threadId; + SRWLatch latch; + SDnodeWorker mgmtWorker; } SDnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 8f4287a539..95ee961dd0 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -21,12 +21,9 @@ #include "dndSnode.h" #include "dndTransport.h" #include "dndVnodes.h" +#include "dndWorker.h" -static int32_t dndInitMgmtWorker(SDnode *pDnode); -static void dndCleanupMgmtWorker(SDnode *pDnode); -static int32_t dndAllocMgmtQueue(SDnode *pDnode); -static void dndFreeMgmtQueue(SDnode *pDnode); -static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); +static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg); static int32_t dndReadDnodes(SDnode *pDnode); static int32_t dndWriteDnodes(SDnode *pDnode); @@ -534,13 +531,8 @@ int32_t dndInitDnode(SDnode *pDnode) { return -1; } - if (dndInitMgmtWorker(pDnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (dndAllocMgmtQueue(pDnode) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + if (dndInitWorker(pDnode, &pMgmt->mgmtWorker, DND_WORKER_SINGLE, "dnode-mgmt", 1, 1, dndProcessMgmtQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); return -1; } @@ -551,15 +543,14 @@ int32_t dndInitDnode(SDnode *pDnode) { return -1; } - dInfo("dnode-dnode is initialized"); + dInfo("dnode-mgmt is initialized"); return 0; } void dndCleanupDnode(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - dndCleanupMgmtWorker(pDnode); - dndFreeMgmtQueue(pDnode); + dndCleanupWorker(&pMgmt->mgmtWorker); if (pMgmt->threadId != NULL) { taosDestoryThread(pMgmt->threadId); @@ -584,62 +575,22 @@ void dndCleanupDnode(SDnode *pDnode) { } taosWUnLockLatch(&pMgmt->latch); - dInfo("dnode-dnode is cleaned up"); + dInfo("dnode-mgmt is cleaned up"); } -static int32_t dndInitMgmtWorker(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - SWorkerPool *pPool = &pMgmt->mgmtPool; - pPool->name = "dnode-mgmt"; - pPool->min = 1; - pPool->max = 1; - if (tWorkerInit(pPool) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - dDebug("dnode mgmt worker is initialized"); - return 0; -} - -static void dndCleanupMgmtWorker(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - tWorkerCleanup(&pMgmt->mgmtPool); - dDebug("dnode mgmt worker is closed"); -} - -static int32_t dndAllocMgmtQueue(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - pMgmt->pMgmtQ = tWorkerAllocQueue(&pMgmt->mgmtPool, pDnode, (FProcessItem)dndProcessMgmtQueue); - if (pMgmt->pMgmtQ == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - return 0; -} - -static void dndFreeMgmtQueue(SDnode *pDnode) { - SDnodeMgmt *pMgmt = &pDnode->dmgmt; - tWorkerFreeQueue(&pMgmt->mgmtPool, pMgmt->pMgmtQ); - pMgmt->pMgmtQ = NULL; -} - -void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) { +void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - if (pEpSet && pEpSet->numOfEps > 0 && pRpcMsg->msgType == TDMT_MND_STATUS_RSP) { + if (pEpSet && pEpSet->numOfEps > 0 && pMsg->msgType == TDMT_MND_STATUS_RSP) { dndUpdateMnodeEpSet(pDnode, pEpSet); } - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg)); - if (pMsg != NULL) *pMsg = *pRpcMsg; - - if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) { - if (pRpcMsg->msgType & 1u) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; + if (dndWriteMsgToWorker(&pMgmt->mgmtWorker, pMsg, sizeof(SRpcMsg)) != 0) { + if (pMsg->msgType & 1u) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; rpcSendResponse(&rsp); } - rpcFreeCont(pRpcMsg->pCont); + rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } } From 767418e31c5da6d9821b3cefc15a4649364c31f2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 6 Jan 2022 03:51:44 -0800 Subject: [PATCH 51/54] rename file --- source/dnode/mgmt/impl/inc/{dndDnode.h => dndMgmt.h} | 0 source/dnode/mgmt/impl/src/dndBnode.c | 2 +- source/dnode/mgmt/impl/src/{dndDnode.c => dndMgmt.c} | 2 +- source/dnode/mgmt/impl/src/dndMnode.c | 2 +- source/dnode/mgmt/impl/src/dndQnode.c | 2 +- source/dnode/mgmt/impl/src/dndSnode.c | 2 +- source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mgmt/impl/src/dnode.c | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename source/dnode/mgmt/impl/inc/{dndDnode.h => dndMgmt.h} (100%) rename source/dnode/mgmt/impl/src/{dndDnode.c => dndMgmt.c} (99%) diff --git a/source/dnode/mgmt/impl/inc/dndDnode.h b/source/dnode/mgmt/impl/inc/dndMgmt.h similarity index 100% rename from source/dnode/mgmt/impl/inc/dndDnode.h rename to source/dnode/mgmt/impl/inc/dndMgmt.h diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 1b2e56edfe..f4774c00df 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "dndBnode.h" -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndTransport.h" #include "dndWorker.h" diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndMgmt.c similarity index 99% rename from source/dnode/mgmt/impl/src/dndDnode.c rename to source/dnode/mgmt/impl/src/dndMgmt.c index 95ee961dd0..7d071be944 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndBnode.h" #include "dndMnode.h" #include "dndQnode.h" diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index a8bf26f133..e947b590ba 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "dndMnode.h" -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndTransport.h" #include "dndWorker.h" diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 845ea8bf9a..31bf5d9b99 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "dndQnode.h" -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndTransport.h" #include "dndWorker.h" diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 7f234629cb..05f3611386 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "dndSnode.h" -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndTransport.h" #include "dndWorker.h" diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index cf0f561609..49b9e8d6e1 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -21,7 +21,7 @@ #define _DEFAULT_SOURCE #include "dndTransport.h" -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndMnode.h" #include "dndVnodes.h" diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index ef5c15743c..362838eba0 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "dndBnode.h" -#include "dndDnode.h" +#include "dndMgmt.h" #include "dndMnode.h" #include "dndQnode.h" #include "dndSnode.h" From 5c73c1ffd8f4a7dfece8953471ee33fef02269f7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 6 Jan 2022 19:53:14 +0800 Subject: [PATCH 52/54] refine heartbeat interface --- source/client/inc/clientHb.h | 145 ++++++++++++++++++----------------- source/client/src/clientHb.c | 11 +-- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h index 676b5c4c40..11fea4de5e 100644 --- a/source/client/inc/clientHb.h +++ b/source/client/inc/clientHb.h @@ -20,6 +20,8 @@ typedef enum { mq = 0, + // type can be added here + // HEARTBEAT_TYPE_MAX } EHbType; @@ -30,80 +32,16 @@ typedef struct SKlv { void* value; } SKlv; -static FORCE_INLINE int taosEncodeSKlv(void** buf, const SKlv* pKlv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKlv->keyLen); - tlen += taosEncodeFixedI32(buf, pKlv->valueLen); - tlen += taosEncodeBinary(buf, pKlv->key, pKlv->keyLen); - tlen += taosEncodeBinary(buf, pKlv->value, pKlv->valueLen); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSKlv(void* buf, SKlv* pKlv) { - buf = taosDecodeFixedI32(buf, &pKlv->keyLen); - buf = taosDecodeFixedI32(buf, &pKlv->valueLen); - buf = taosDecodeBinary(buf, &pKlv->key, pKlv->keyLen); - buf = taosDecodeBinary(buf, &pKlv->value, pKlv->valueLen); - return buf; -} - typedef struct SClientHbKey { int32_t connId; int32_t hbType; } SClientHbKey; -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; -} - typedef struct SClientHbReq { SClientHbKey hbKey; - SHashObj* info; // hash + SHashObj* info; // hash } SClientHbReq; -static FORCE_INLINE int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq) { - int tlen = 0; - tlen += taosEncodeSClientHbKey(buf, &pReq->hbKey); - - void* pIter = NULL; - void* data; - SKlv klv; - data = taosHashIterate(pReq->info, pIter); - while (data != NULL) { - taosHashGetKey(data, &klv.key, (size_t*)&klv.keyLen); - klv.valueLen = taosHashGetDataLen(data); - klv.value = data; - taosEncodeSKlv(buf, &klv); - - data = taosHashIterate(pReq->info, pIter); - } - return tlen; -} - -static FORCE_INLINE void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq) { - ASSERT(pReq->info != NULL); - buf = taosDecodeSClientHbKey(buf, &pReq->hbKey); - - //TODO: error handling - if(pReq->info == NULL) { - pReq->info = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - } - SKlv klv; - buf = taosDecodeSKlv(buf, &klv); - taosHashPut(pReq->info, klv.key, klv.keyLen, klv.value, klv.valueLen); - - return buf; -} - typedef struct SClientHbBatchReq { int64_t reqId; SArray* reqs; // SArray @@ -123,15 +61,16 @@ typedef struct SClientHbBatchRsp { SArray* rsps; // SArray } SClientHbBatchRsp; -typedef int32_t (*FHbRspHandle)(SClientHbReq* pReq); -typedef int32_t (*FGetConnInfo)(int32_t conn, void* self); +typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); +typedef int32_t (*FGetConnInfo)(SClientHbKey connKey, void* param); typedef struct SClientHbMgr { int8_t inited; int32_t reportInterval; // unit ms int32_t stats; SRWLatch lock; - SHashObj* info; //hash + SHashObj* activeInfo; // hash + SHashObj* getInfoFuncs; // hash FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; // input queue } SClientHbMgr; @@ -141,8 +80,72 @@ static SClientHbMgr clientHbMgr = {0}; int hbMgrInit(); void hbMgrCleanUp(); -int registerConn(int32_t connId, FGetConnInfo func, FHbRspHandle rspHandle); +int hbHandleRsp(void* hbMsg); -int registerHbRspHandle(int32_t connId, int32_t hbType, FHbRspHandle rspHandle); +int hbRegisterConn(SClientHbKey connKey, FGetConnInfo func); -int HbAddConnInfo(int32_t connId, void* key, void* value, int32_t keyLen, int32_t valueLen); +int hbAddConnInfo(SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); + +static FORCE_INLINE int taosEncodeSKlv(void** buf, const SKlv* pKlv) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKlv->keyLen); + tlen += taosEncodeFixedI32(buf, pKlv->valueLen); + tlen += taosEncodeBinary(buf, pKlv->key, pKlv->keyLen); + tlen += taosEncodeBinary(buf, pKlv->value, pKlv->valueLen); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSKlv(void* buf, SKlv* pKlv) { + buf = taosDecodeFixedI32(buf, &pKlv->keyLen); + buf = taosDecodeFixedI32(buf, &pKlv->valueLen); + buf = taosDecodeBinary(buf, &pKlv->key, pKlv->keyLen); + buf = taosDecodeBinary(buf, &pKlv->value, pKlv->valueLen); + return buf; +} + +static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKey->connId); + tlen += taosEncodeFixedI32(buf, pKey->hbType); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { + buf = taosDecodeFixedI32(buf, &pKey->connId); + buf = taosDecodeFixedI32(buf, &pKey->hbType); + return buf; +} + +static FORCE_INLINE int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq) { + int tlen = 0; + tlen += taosEncodeSClientHbKey(buf, &pReq->hbKey); + + void* pIter = NULL; + void* data; + SKlv klv; + data = taosHashIterate(pReq->info, pIter); + while (data != NULL) { + taosHashGetKey(data, &klv.key, (size_t*)&klv.keyLen); + klv.valueLen = taosHashGetDataLen(data); + klv.value = data; + taosEncodeSKlv(buf, &klv); + + data = taosHashIterate(pReq->info, pIter); + } + return tlen; +} + +static FORCE_INLINE void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq) { + ASSERT(pReq->info != NULL); + buf = taosDecodeSClientHbKey(buf, &pReq->hbKey); + + // TODO: error handling + if (pReq->info == NULL) { + pReq->info = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + SKlv klv; + buf = taosDecodeSKlv(buf, &klv); + taosHashPut(pReq->info, klv.key, klv.keyLen, klv.value, klv.valueLen); + + return buf; +} diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f1f7409f05..1de295384b 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -15,7 +15,7 @@ #include "clientHb.h" -static int32_t mqHbRspHandle(SClientHbReq* pReq) { +static int32_t mqHbRspHandle(SClientHbRsp* pReq) { return 0; } @@ -42,15 +42,12 @@ void hbMgrCleanUp() { } -int registerConn(int32_t connId, FGetConnInfo func, FHbRspHandle rspHandle) { +int hbRegisterConn(SClientHbKey connKey, FGetConnInfo func) { + return 0; } -int registerHbRspHandle(int32_t connId, int32_t hbType, FHbRspHandle rspHandle) { - return 0; -} - -int HbAddConnInfo(int32_t connId, void* key, void* value, int32_t keyLen, int32_t valueLen) { +int hbAddConnInfo(SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { //lock //find req by connection id From 11449f3d988136bc7429ba5a0c5fa540caaf098f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 6 Jan 2022 19:58:54 +0800 Subject: [PATCH 53/54] refine heartbeat interface --- source/client/inc/clientHb.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h index 11fea4de5e..4ab11239b7 100644 --- a/source/client/inc/clientHb.h +++ b/source/client/inc/clientHb.h @@ -38,7 +38,7 @@ typedef struct SClientHbKey { } SClientHbKey; typedef struct SClientHbReq { - SClientHbKey hbKey; + SClientHbKey connKey; SHashObj* info; // hash } SClientHbReq; @@ -51,8 +51,10 @@ typedef struct SClientHbHandleResult { } SClientHbHandleResult; typedef struct SClientHbRsp { - int32_t connId; - int32_t hbType; + SClientHbKey connKey; + int32_t status; + int32_t bodyLen; + void* body; } SClientHbRsp; typedef struct SClientHbBatchRsp { @@ -118,7 +120,7 @@ static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) static FORCE_INLINE int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq) { int tlen = 0; - tlen += taosEncodeSClientHbKey(buf, &pReq->hbKey); + tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); void* pIter = NULL; void* data; @@ -137,7 +139,7 @@ static FORCE_INLINE int tSerializeSClientHbReq(void** buf, const SClientHbReq* p static FORCE_INLINE void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq) { ASSERT(pReq->info != NULL); - buf = taosDecodeSClientHbKey(buf, &pReq->hbKey); + buf = taosDecodeSClientHbKey(buf, &pReq->connKey); // TODO: error handling if (pReq->info == NULL) { From 7583b45ecb263e69526c4a0bef1deec2d9d4b2f6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 7 Jan 2022 10:36:42 +0800 Subject: [PATCH 54/54] refine heartbeat interface --- include/common/tmsg.h | 80 +++++++++++++++++++++++++-- source/client/inc/clientHb.h | 104 +---------------------------------- source/client/src/clientHb.c | 36 +++++++++--- source/common/src/tmsg.c | 36 +++++++++++- 4 files changed, 139 insertions(+), 117 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7d232aa852..954f24ef6a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -26,6 +26,7 @@ extern "C" { #include "tarray.h" #include "tcoding.h" #include "tdataformat.h" +#include "thash.h" #include "tlist.h" /* ------------------------ MESSAGE DEFINITIONS ------------------------ */ @@ -132,6 +133,73 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) +typedef struct SKlv { + int32_t keyLen; + int32_t valueLen; + void* key; + void* value; +} SKlv; + +static FORCE_INLINE int taosEncodeSKlv(void** buf, const SKlv* pKlv) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKlv->keyLen); + tlen += taosEncodeFixedI32(buf, pKlv->valueLen); + tlen += taosEncodeBinary(buf, pKlv->key, pKlv->keyLen); + tlen += taosEncodeBinary(buf, pKlv->value, pKlv->valueLen); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSKlv(void* buf, SKlv* pKlv) { + buf = taosDecodeFixedI32(buf, &pKlv->keyLen); + buf = taosDecodeFixedI32(buf, &pKlv->valueLen); + buf = taosDecodeBinary(buf, &pKlv->key, pKlv->keyLen); + buf = taosDecodeBinary(buf, &pKlv->value, pKlv->valueLen); + return buf; +} +typedef struct SClientHbKey { + int32_t connId; + int32_t hbType; +} SClientHbKey; + +static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pKey->connId); + tlen += taosEncodeFixedI32(buf, pKey->hbType); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { + buf = taosDecodeFixedI32(buf, &pKey->connId); + buf = taosDecodeFixedI32(buf, &pKey->hbType); + return buf; +} + +typedef struct SClientHbReq { + SClientHbKey connKey; + SHashObj* info; // hash +} SClientHbReq; + +typedef struct SClientHbBatchReq { + int64_t reqId; + SArray* reqs; // SArray +} SClientHbBatchReq; + +int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); +void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq); + +typedef struct SClientHbRsp { + SClientHbKey connKey; + int32_t status; + int32_t bodyLen; + void* body; +} SClientHbRsp; + +typedef struct SClientHbBatchRsp { + int64_t reqId; + int64_t rspId; + SArray* rsps; // SArray +} SClientHbBatchRsp; + typedef struct SBuildTableMetaInput { int32_t vgId; char* dbName; @@ -1126,7 +1194,7 @@ typedef struct { int32_t topicNum; int64_t consumerId; char* consumerGroup; - SArray* topicNames; // SArray + SArray* topicNames; // SArray } SCMSubscribeReq; static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -1135,7 +1203,7 @@ static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribe tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->consumerGroup); - for(int i = 0; i < pReq->topicNum; i++) { + for (int i = 0; i < pReq->topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } return tlen; @@ -1146,7 +1214,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeString(buf, &pReq->consumerGroup); pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); - for(int i = 0; i < pReq->topicNum; i++) { + for (int i = 0; i < pReq->topicNum; i++) { char* name = NULL; buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); @@ -1161,14 +1229,14 @@ typedef struct SMqSubTopic { } SMqSubTopic; typedef struct { - int32_t topicNum; + int32_t topicNum; SMqSubTopic topics[]; } SCMSubscribeRsp; static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { int tlen = 0; tlen += taosEncodeFixedI32(buf, pRsp->topicNum); - for(int i = 0; i < pRsp->topicNum; i++) { + for (int i = 0; i < pRsp->topicNum; i++) { tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId); tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId); tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet); @@ -1178,7 +1246,7 @@ static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribe static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { buf = taosDecodeFixedI32(buf, &pRsp->topicNum); - for(int i = 0; i < pRsp->topicNum; i++) { + for (int i = 0; i < pRsp->topicNum; i++) { buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId); buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId); buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet); diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h index 4ab11239b7..73adb41308 100644 --- a/source/client/inc/clientHb.h +++ b/source/client/inc/clientHb.h @@ -25,44 +25,6 @@ typedef enum { HEARTBEAT_TYPE_MAX } EHbType; -typedef struct SKlv { - int32_t keyLen; - int32_t valueLen; - void* key; - void* value; -} SKlv; - -typedef struct SClientHbKey { - int32_t connId; - int32_t hbType; -} SClientHbKey; - -typedef struct SClientHbReq { - SClientHbKey connKey; - SHashObj* info; // hash -} SClientHbReq; - -typedef struct SClientHbBatchReq { - int64_t reqId; - SArray* reqs; // SArray -} SClientHbBatchReq; - -typedef struct SClientHbHandleResult { -} SClientHbHandleResult; - -typedef struct SClientHbRsp { - SClientHbKey connKey; - int32_t status; - int32_t bodyLen; - void* body; -} SClientHbRsp; - -typedef struct SClientHbBatchRsp { - int64_t reqId; - int64_t rspId; - SArray* rsps; // SArray -} SClientHbBatchRsp; - typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); typedef int32_t (*FGetConnInfo)(SClientHbKey connKey, void* param); @@ -81,73 +43,11 @@ static SClientHbMgr clientHbMgr = {0}; int hbMgrInit(); void hbMgrCleanUp(); - int hbHandleRsp(void* hbMsg); + int hbRegisterConn(SClientHbKey connKey, FGetConnInfo func); + int hbAddConnInfo(SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); -static FORCE_INLINE int taosEncodeSKlv(void** buf, const SKlv* pKlv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKlv->keyLen); - tlen += taosEncodeFixedI32(buf, pKlv->valueLen); - tlen += taosEncodeBinary(buf, pKlv->key, pKlv->keyLen); - tlen += taosEncodeBinary(buf, pKlv->value, pKlv->valueLen); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSKlv(void* buf, SKlv* pKlv) { - buf = taosDecodeFixedI32(buf, &pKlv->keyLen); - buf = taosDecodeFixedI32(buf, &pKlv->valueLen); - buf = taosDecodeBinary(buf, &pKlv->key, pKlv->keyLen); - buf = taosDecodeBinary(buf, &pKlv->value, pKlv->valueLen); - return buf; -} - -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; -} - -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; -} - -static FORCE_INLINE int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq) { - int tlen = 0; - tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); - - void* pIter = NULL; - void* data; - SKlv klv; - data = taosHashIterate(pReq->info, pIter); - while (data != NULL) { - taosHashGetKey(data, &klv.key, (size_t*)&klv.keyLen); - klv.valueLen = taosHashGetDataLen(data); - klv.value = data; - taosEncodeSKlv(buf, &klv); - - data = taosHashIterate(pReq->info, pIter); - } - return tlen; -} - -static FORCE_INLINE void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq) { - ASSERT(pReq->info != NULL); - buf = taosDecodeSClientHbKey(buf, &pReq->connKey); - - // TODO: error handling - if (pReq->info == NULL) { - pReq->info = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - } - SKlv klv; - buf = taosDecodeSKlv(buf, &klv); - taosHashPut(pReq->info, klv.key, klv.keyLen, klv.value, klv.valueLen); - - return buf; -} diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1de295384b..7daa1204d0 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -19,27 +19,44 @@ static int32_t mqHbRspHandle(SClientHbRsp* pReq) { return 0; } +uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { + return 0; +} + +static void hbMgrInitMqHbFunc() { + clientHbMgr.handle[mq] = mqHbRspHandle; +} + int hbMgrInit() { //init once - // - //init lock - // - //init handle funcs - clientHbMgr.handle[mq] = mqHbRspHandle; + int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); + if (old == 1) return 0; + + //init config + clientHbMgr.reportInterval = 1500; //init stat clientHbMgr.stats = 0; - //init config - clientHbMgr.reportInterval = 1500; + //init lock + taosInitRWLatch(&clientHbMgr.lock); + + //init handle funcs + hbMgrInitMqHbFunc(); //init hash info - // + clientHbMgr.activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + //init getInfoFunc + clientHbMgr.getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); return 0; } void hbMgrCleanUp() { + int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); + if (old == 0) return; + taosHashCleanup(clientHbMgr.activeInfo); + taosHashCleanup(clientHbMgr.getInfoFuncs); } int hbRegisterConn(SClientHbKey connKey, FGetConnInfo func) { @@ -51,6 +68,9 @@ int hbAddConnInfo(SClientHbKey connKey, void* key, void* value, int32_t keyLen, //lock //find req by connection id + SClientHbReq* data = taosHashGet(clientHbMgr.activeInfo, &connKey, sizeof(SClientHbKey)); + ASSERT(data != NULL); + taosHashPut(data->info, key, keyLen, value, valueLen); //unlock return 0; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c1048f8482..5cbb42c1e6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,6 +27,40 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { + int tlen = 0; + tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); + + void *pIter = NULL; + void *data; + SKlv klv; + data = taosHashIterate(pReq->info, pIter); + while (data != NULL) { + taosHashGetKey(data, &klv.key, (size_t *)&klv.keyLen); + klv.valueLen = taosHashGetDataLen(data); + klv.value = data; + taosEncodeSKlv(buf, &klv); + + data = taosHashIterate(pReq->info, pIter); + } + return tlen; +} + +void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) { + ASSERT(pReq->info != NULL); + buf = taosDecodeSClientHbKey(buf, &pReq->connKey); + + // TODO: error handling + if (pReq->info == NULL) { + pReq->info = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + SKlv klv; + buf = taosDecodeSKlv(buf, &klv); + taosHashPut(pReq->info, klv.key, klv.keyLen, klv.value, klv.valueLen); + + return buf; +} + int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int tlen = 0; @@ -148,4 +182,4 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { } return buf; -} \ No newline at end of file +}