From 3f3fb4a27168c8d42262a53833ebda0f3ee20734 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 28 Dec 2021 17:44:01 +0800 Subject: [PATCH 01/97] add node by manual operation --- contrib/test/craft/common.h | 1 + contrib/test/craft/raftMain.c | 76 +++++++++++++++++++++++++++------ contrib/test/craft/raftServer.c | 26 ++++++++--- 3 files changed, 85 insertions(+), 18 deletions(-) diff --git a/contrib/test/craft/common.h b/contrib/test/craft/common.h index 1e94ee8bca..282e282543 100644 --- a/contrib/test/craft/common.h +++ b/contrib/test/craft/common.h @@ -20,6 +20,7 @@ typedef struct { } Addr; typedef struct { + int voter; Addr me; Addr peers[MAX_PEERS]; int peersCount; diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c index 52e0b694dc..bae083cf94 100644 --- a/contrib/test/craft/raftMain.c +++ b/contrib/test/craft/raftMain.c @@ -104,7 +104,7 @@ const char* state2String(unsigned short state) { void printRaftConfiguration(struct raft_configuration *c) { printf("configuration: \n"); for (int i = 0; i < c->n; ++i) { - printf("%llu -- %d -- %s\n", c->servers->id, c->servers->role, c->servers->address); + printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); } } @@ -119,11 +119,9 @@ void printRaftState(struct raft *r) { printf("last_applied: %llu \n", r->last_applied); printf("last_stored: %llu \n", r->last_stored); - /* printf("configuration_index: %llu \n", r->configuration_index); printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); printRaftConfiguration(&r->configuration); - */ printf("----------------------------\n"); } @@ -164,6 +162,18 @@ void getValue(const char *key) { } } +void raft_change_cb_add(struct raft_change *req, int status) { + printf("raft_change_cb_add status:%d ... \n", status); +} + +void raft_change_cb_assign(struct raft_change *req, int status) { + printf("raft_change_cb_assign status:%d ... \n", status); +} + +void raft_change_cb_remove(struct raft_change *req, int status) { + printf("raft_change_cb_remove status:%d ... \n", status); +} + void console(SRaftServer *pRaftServer) { while (1) { char cmd_buf[COMMAND_LEN]; @@ -193,30 +203,59 @@ void console(SRaftServer *pRaftServer) { parseCommand(cmd_buf, cmd, param1, param2, TOKEN_LEN); if (strcmp(cmd, "addnode") == 0) { - printf("not support \n"); + //printf("not support \n"); - /* char host[HOST_LEN]; uint32_t port; parseAddr(param1, host, HOST_LEN, &port); uint64_t rid = raftId(host, port); struct raft_change *req = raft_malloc(sizeof(*req)); - int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL); + int r = raft_add(&pRaftServer->raft, req, rid, param1, raft_change_cb_add); if (r != 0) { - printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft)); + printf("raft_add error: %s \n", raft_errmsg(&pRaftServer->raft)); } printf("add node: %lu %s \n", rid, param1); struct raft_change *req2 = raft_malloc(sizeof(*req2)); - r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL); + r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, raft_change_cb_assign); if (r != 0) { - printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft)); + printf("raft_assign error: %s \n", raft_errmsg(&pRaftServer->raft)); } - */ + printf("raft_assign: %s %d \n", param1, RAFT_VOTER); + + } else if (strcmp(cmd, "activate") == 0) { + char host[HOST_LEN]; + uint32_t port; + parseAddr(param1, host, HOST_LEN, &port); + uint64_t rid = raftId(host, port); + + + struct raft_change *req2 = raft_malloc(sizeof(*req2)); + int r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, raft_change_cb_assign); + if (r != 0) { + printf("raft_assign error: %s \n", raft_errmsg(&pRaftServer->raft)); + } + printf("raft_assign: %s %d \n", param1, RAFT_VOTER); + + + + } else if (strcmp(cmd, "dropnode") == 0) { - printf("not support \n"); + char host[HOST_LEN]; + uint32_t port; + parseAddr(param1, host, HOST_LEN, &port); + uint64_t rid = raftId(host, port); + + struct raft_change *req = raft_malloc(sizeof(*req)); + int r = raft_remove(&pRaftServer->raft, req, rid, raft_change_cb_remove); + if (r != 0) { + printf("raft_remove: %s \n", raft_errmsg(&pRaftServer->raft)); + } + printf("drop node: %lu %s \n", rid, param1); + + } else if (strcmp(cmd, "put") == 0) { char buf[256]; @@ -234,6 +273,7 @@ void console(SRaftServer *pRaftServer) { } else if (strcmp(cmd, "help") == 0) { printf("addnode \"127.0.0.1:8888\" \n"); + printf("activate \"127.0.0.1:8888\" \n"); printf("dropnode \"127.0.0.1:8888\" \n"); printf("put key value \n"); printf("get key \n"); @@ -256,7 +296,9 @@ void *startConsoleFunc(void *param) { // Config --------------------------------- void usage() { printf("\nusage: \n"); - printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10000 --dir=./data --voter \n", exe_name); + printf("%s --me=127.0.0.1:10001 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10002 --dir=./data \n", exe_name); printf("\n"); printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name); printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name); @@ -271,13 +313,15 @@ void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { option_index = 0; static struct option long_options[] = { {"help", no_argument, NULL, 'h'}, + {"voter", no_argument, NULL, 'v'}, {"peers", required_argument, NULL, 'p'}, {"me", required_argument, NULL, 'm'}, {"dir", required_argument, NULL, 'd'}, {NULL, 0, NULL, 0} }; - while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) { + pConf->voter = 0; + while ((option_value = getopt_long(argc, argv, "hvp:m:d:", long_options, &option_index)) != -1) { switch (option_value) { case 'm': { parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); @@ -295,6 +339,10 @@ void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { break; } + case 'v': { + pConf->voter = 1; + break; + } case 'd': { snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg); @@ -338,6 +386,8 @@ int main(int argc, char **argv) { exit(-1); } + signal(SIGPIPE, SIG_IGN); + SRaftServerConfig conf; parseConf(argc, argv, &conf); printConf(&conf); diff --git a/contrib/test/craft/raftServer.c b/contrib/test/craft/raftServer.c index 6f4dbc1997..ffec22e646 100644 --- a/contrib/test/craft/raftServer.c +++ b/contrib/test/craft/raftServer.c @@ -85,29 +85,45 @@ int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, pRaftServer->fsm = pFsm; ret = uv_loop_init(&pRaftServer->loop); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } ret = raft_uv_init(&pRaftServer->io, &pRaftServer->loop, pRaftServer->dir, &pRaftServer->transport); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } ret = raft_init(&pRaftServer->raft, &pRaftServer->io, pRaftServer->fsm, pRaftServer->raftId, pRaftServer->address); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } struct raft_configuration conf; raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_VOTER); + + if (pConf->voter == 0) { + raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_SPARE); + + } else { + raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_VOTER); + + } + + + printf("add myself: %llu - %s \n", pRaftServer->raftId, pRaftServer->address); + + for (int i = 0; i < pConf->peersCount; ++i) { const Addr *pAddr = &pConf->peers[i]; raft_id rid = raftId(pAddr->host, pAddr->port); From c7bb4eace67d86b4dab70b40a7315c83e8612aac Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Dec 2021 09:51:06 +0800 Subject: [PATCH 02/97] refactor code --- source/libs/index/test/indexTests.cc | 33 +++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index b3e385192f..110499f67a 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -725,23 +725,40 @@ TEST_F(IndexEnv2, testIndexOpen) { assert(taosArrayGetSize(result) == targetSize); } } -TEST_F(IndexEnv2, testIndex_CachePut) { + +TEST_F(IndexEnv2, testIndex_TrigeFlush) { + std::string path = "/tmp"; + if (index->Init(path) != 0) { + } +} +TEST_F(IndexEnv2, testIndex_TrigeFlush) { + std::string path = "/tmp"; + if (index->Init(path) != 0) { + } +} +TEST_(IndexEnv2, testIndex_serarch_cache_and_tfile) { + std::string path = "/tmp"; + if (index->Init(path) != 0) { + } +} +TEST_(IndexEnv2, testIndex_multi_thread_write) { + std::string path = "/tmp"; + if (index->Init(path) != 0) { + } +} +TEST_(IndexEnv2, testIndex_multi_thread_read) { std::string path = "/tmp"; if (index->Init(path) != 0) { } } -TEST_F(IndexEnv2, testIndexr_TFilePut) { +TEST_(IndexEnv2, testIndex_restart) { std::string path = "/tmp"; if (index->Init(path) != 0) { } } -TEST_F(IndexEnv2, testIndex_CacheSearch) { - std::string path = "/tmp"; - if (index->Init(path) != 0) { - } -} -TEST_F(IndexEnv2, testIndex_TFileSearch) { + +TEST_F(IndexEnv2, testIndex_performance) { std::string path = "/tmp"; if (index->Init(path) != 0) { } From a011e21eead2db6e1dd377a4f887a49572960432 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 00:47:21 -0800 Subject: [PATCH 03/97] minor changes --- source/dnode/mnode/impl/src/mndVgroup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 06e62d2528..78205d5f6a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -273,7 +273,7 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) { static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; - return d1Score > d2Score ? 0 : 1; + return d1Score > d2Score ? 1 : 0; } static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { From a514dfe4707fb26efa7fdb69d4a1f11eaa207b06 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 29 Dec 2021 09:20:10 +0000 Subject: [PATCH 04/97] more work --- include/common/tmsg.h | 128 ++++++++---------------------- source/common/src/tmsg.c | 124 ++++++++++++++++++++++++++++- source/common/test/CMakeLists.txt | 16 ++-- 3 files changed, 166 insertions(+), 102 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b386b729d4..49d0f9c97c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -25,7 +25,9 @@ extern "C" { #include "taoserror.h" #include "tcoding.h" #include "tdataformat.h" +#include "tlist.h" +/* ------------------------ MESSAGE DEFINITIONS ------------------------ */ #define TD_MSG_NUMBER_ #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ @@ -54,6 +56,35 @@ extern int tMsgDict[]; typedef uint16_t tmsg_t; +/* ------------------------ ENCODE/DECODE FUNCTIONS AND MACROS ------------------------ */ +typedef struct SMsgEncoder { + SEncoder coder; +} SMsgEncoder; + +struct SMDFreeListNode { + TD_SLIST_NODE(SMDFreeListNode); + char payload[]; +}; + +typedef struct SMsgDecoder { + SDecoder coder; + TD_SLIST(SMDFreeListNode) freeList; +} SMsgDecoder; + +#define TMSG_MALLOC(SIZE, DECODER) \ + ({ \ + void* ptr = malloc((SIZE) + sizeof(struct SMDFreeListNode)); \ + if (ptr) { \ + TD_SLIST_PUSH(&((DECODER)->freeList), (struct SMDFreeListNode*)ptr); \ + ptr = POINTER_SHIFT(ptr, sizeof(struct SMDFreeListNode*)); \ + } \ + ptr; \ + }) + +void tmsgInitMsgDecoder(SMsgDecoder* pMD, td_endian_t endian, uint8_t* data, int64_t size); +void tmsgClearMsgDecoder(SMsgDecoder* pMD); + +/* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 #define TSDB_IE_TYPE_META 2 @@ -1143,100 +1174,11 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -static FORCE_INLINE int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq) { - int tlen = 0; +int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); +int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); +int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq); +void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); - tlen += taosEncodeFixedU64(buf, pReq->ver); - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedU32(buf, pReq->ttl); - tlen += taosEncodeFixedU32(buf, pReq->keep); - tlen += taosEncodeFixedU8(buf, pReq->type); - - switch (pReq->type) { - case TD_SUPER_TABLE: - tlen += taosEncodeFixedU64(buf, pReq->stbCfg.suid); - tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); - for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); - } - tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); - for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); - } - break; - case TD_CHILD_TABLE: - tlen += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); - tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); - break; - case TD_NORMAL_TABLE: - tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); - for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { - tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); - } - break; - default: - ASSERT(0); - } - - return tlen; -} - -static FORCE_INLINE void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeFixedU32(buf, &(pReq->ttl)); - buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->type)); - - switch (pReq->type) { - case TD_SUPER_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); - buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); - pReq->stbCfg.pSchema = (SSchema*)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); - buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); - } - buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); - pReq->stbCfg.pTagSchema = (SSchema*)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); - buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); - } - break; - case TD_CHILD_TABLE: - buf = taosDecodeFixedU64(buf, &pReq->ctbCfg.suid); - buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); - break; - case TD_NORMAL_TABLE: - buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); - pReq->ntbCfg.pSchema = (SSchema*)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { - buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); - buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); - } - break; - default: - ASSERT(0); - } - - return buf; -} typedef struct SVCreateTbRsp { } SVCreateTbRsp; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7f285046c..4cabd1bb34 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -25,4 +25,126 @@ #undef TD_MSG_INFO_ #define TD_MSG_DICT_ #undef TD_MSG_SEG_CODE_ -#include "tmsgdef.h" \ No newline at end of file +#include "tmsgdef.h" + +/* ------------------------ ENCODE/DECODE FUNCTIONS ------------------------ */ +void tmsgInitMsgDecoder(SMsgDecoder *pMD, td_endian_t endian, uint8_t *data, int64_t size) { + tInitDecoder(&pMD->coder, endian, data, size); + TD_SLIST_INIT(&(pMD->freeList)); +} + +void tmsgClearMsgDecoder(SMsgDecoder *pMD) { + struct SMDFreeListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pMD->freeList)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pMD->freeList)); + free(pNode); + } +} + +/* ------------------------ MESSAGE ENCODE/DECODE ------------------------ */ +int tmsgSVCreateTbReqEncode(SMsgEncoder *pCoder, SVCreateTbReq *pReq) { + // TODO + return 0; +} + +int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) { + // TODO + return 0; +} + +int tSerializeSVCreateTbReq(void **buf, const SVCreateTbReq *pReq) { + int tlen = 0; + + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU32(buf, pReq->ttl); + tlen += taosEncodeFixedU32(buf, pReq->keep); + tlen += taosEncodeFixedU8(buf, pReq->type); + + switch (pReq->type) { + case TD_SUPER_TABLE: + tlen += taosEncodeFixedU64(buf, pReq->stbCfg.suid); + tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); + for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); + } + tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); + for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); + } + break; + case TD_CHILD_TABLE: + tlen += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); + tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); + break; + case TD_NORMAL_TABLE: + tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); + for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); + } + break; + default: + ASSERT(0); + } + + return tlen; +} + +void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { + buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeString(buf, &(pReq->name)); + buf = taosDecodeFixedU32(buf, &(pReq->ttl)); + buf = taosDecodeFixedU32(buf, &(pReq->keep)); + buf = taosDecodeFixedU8(buf, &(pReq->type)); + + switch (pReq->type) { + case TD_SUPER_TABLE: + buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); + buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); + pReq->stbCfg.pSchema = (SSchema *)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); + buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); + buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); + buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); + } + buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); + pReq->stbCfg.pTagSchema = (SSchema *)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); + buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); + buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); + } + break; + case TD_CHILD_TABLE: + buf = taosDecodeFixedU64(buf, &pReq->ctbCfg.suid); + buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); + break; + case TD_NORMAL_TABLE: + buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); + pReq->ntbCfg.pSchema = (SSchema *)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); + buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); + buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); + } + break; + default: + ASSERT(0); + } + + return buf; +} \ No newline at end of file diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index 03ca95ff64..58dde913f0 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -18,11 +18,11 @@ TARGET_INCLUDE_DIRECTORIES( ) # tmsg test -add_executable(tmsgTest "") -target_sources(tmsgTest - PRIVATE - "tmsgTest.cpp" - "../src/tmsg.c" -) -target_include_directories(tmsgTest PUBLIC "${CMAKE_SOURCE_DIR}/include/common/") -target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) \ No newline at end of file +# add_executable(tmsgTest "") +# target_sources(tmsgTest +# PRIVATE +# "tmsgTest.cpp" +# "../src/tmsg.c" +# ) +# target_include_directories(tmsgTest PUBLIC "${CMAKE_SOURCE_DIR}/include/common/") +# target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) \ No newline at end of file From d61aa9b6d1f2bac20218f2a17384938b57207409 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Dec 2021 17:45:14 +0800 Subject: [PATCH 05/97] feature/qnode --- include/libs/catalog/catalog.h | 12 + include/libs/qcom/query.h | 1 + include/util/thash.h | 30 +++ source/libs/catalog/inc/catalogInt.h | 31 +++ source/libs/catalog/src/catalog.c | 315 ++++++++++++++++++--------- source/libs/qworker/inc/qworkerInt.h | 8 +- source/util/CMakeLists.txt | 4 +- source/util/src/thash.c | 21 +- source/util/test/CMakeLists.txt | 11 +- source/util/test/hashTest.cpp | 54 ++++- 10 files changed, 375 insertions(+), 112 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index a9abf45c8d..d8f14f2a95 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -61,6 +61,18 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); +/** + * Get a DB's all vgroup info. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param forceUpdate (input, force update db vgroup info from mnode) + * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) + * @return error code + */ +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, int32_t forceUpdate, SArray** pVgroupList); + int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); /** diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index ea131bbbf2..c8e81a2e94 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -75,6 +75,7 @@ typedef struct STableMeta { typedef struct SDBVgroupInfo { + int32_t lock; int32_t vgVersion; int8_t hashMethod; SHashObj *vgInfo; //key:vgId, value:SVgroupInfo diff --git a/include/util/thash.h b/include/util/thash.h index d0247a0729..f38ab50893 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -144,6 +144,16 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); */ void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* destBuf); +/** + * Clone the result to interval allocated buffer + * @param pHashObj + * @param key + * @param keyLen + * @param destBuf + * @return + */ +void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz); + /** * remove item with the specified key * @param pHashObj @@ -200,6 +210,26 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); */ int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); +/** + * return the payload data with the specified key(reference number added) + * + * @param pHashObj + * @param key + * @param keyLen + * @return + */ +void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); + +/** + * release the prevous acquired obj + * + * @param pHashObj + * @param data + * @return + */ +void taosHashRelease(SHashObj *pHashObj, void *p); + + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 40943849f1..31b5939463 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -31,6 +31,11 @@ extern "C" { #define CTG_DEFAULT_INVALID_VERSION (-1) +enum { + CTG_READ = 1, + CTG_WRITE, +}; + typedef struct SVgroupListCache { int32_t vgroupVersion; SHashObj *cache; // key:vgId, value:SVgroupInfo @@ -41,6 +46,7 @@ typedef struct SDBVgroupCache { } SDBVgroupCache; typedef struct STableMetaCache { + SRWLatch stableLock; SHashObj *cache; //key:fulltablename, value:STableMeta SHashObj *stableCache; //key:suid, value:STableMeta* } STableMetaCache; @@ -71,6 +77,31 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0) #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +#define CTG_LOCK(type, _lock) do { \ + if (CTG_READ == (type)) { \ + if ((*(_lock)) < 0) assert(0); \ + taosRLockLatch(_lock); \ + ctgDebug("CTG RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } else { \ + if ((*(_lock)) < 0) assert(0); \ + taosWLockLatch(_lock); \ + ctgDebug("CTG WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } \ +} while (0) + +#define CTG_UNLOCK(type, _lock) do { \ + if (CTG_READ == (type)) { \ + if ((*(_lock)) <= 0) assert(0); \ + taosRUnLockLatch(_lock); \ + ctgDebug("CTG RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } else { \ + if ((*(_lock)) <= 0) assert(0); \ + taosWUnLockLatch(_lock); \ + ctgDebug("CTG WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + } \ +} while (0) + + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index e067a7597a..57a8882f4c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -20,24 +20,28 @@ SCatalogMgmt ctgMgmt = {0}; -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo *dbInfo, int32_t *exist) { +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, bool *inCache) { if (NULL == pCatalog->dbCache.cache) { - *exist = 0; + *inCache = false; return TSDB_CODE_SUCCESS; } - SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + SDBVgroupInfo *info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); if (NULL == info) { - *exist = 0; + *inCache = false; return TSDB_CODE_SUCCESS; } - if (dbInfo) { - *dbInfo = *info; + CTG_LOCK(CTG_READ, &info->lock); + if (NULL == info->vgInfo) { + CTG_UNLOCK(CTG_READ, &info->lock); + *inCache = false; + return TSDB_CODE_SUCCESS; } - *exist = 1; + *dbInfo = info; + *inCache = true; return TSDB_CODE_SUCCESS; } @@ -81,46 +85,51 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, snprintf(tbFullName, sizeof(tbFullName), "%s.%s", dbName, pTableName); - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); + *pTableMeta = NULL; - if (NULL == tbMeta) { + size_t sz = 0; + STableMeta *tbMeta = taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)pTableMeta, &sz); + + if (NULL == *pTableMeta) { *exist = 0; return TSDB_CODE_SUCCESS; } - if (tbMeta->tableType == TSDB_CHILD_TABLE) { - STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - *exist = 0; - return TSDB_CODE_SUCCESS; - } + *exist = 1; - if ((*stbMeta)->suid != tbMeta->suid) { - ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); - *pTableMeta = calloc(1, metaSize); - if (NULL == *pTableMeta) { - ctgError("calloc size[%d] failed", metaSize); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta)); - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - } else { - int32_t metaSize = sizeof(STableMeta) + (tbMeta->tableInfo.numOfTags + tbMeta->tableInfo.numOfColumns) * sizeof(SSchema); - *pTableMeta = calloc(1, metaSize); - if (NULL == *pTableMeta) { - ctgError("calloc size[%d] failed", metaSize); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - memcpy(*pTableMeta, tbMeta, metaSize); + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pCatalog->tableCache.stableLock); + + STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); + if (NULL == stbMeta || NULL == *stbMeta) { + CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + qError("no stable:%"PRIx64 " meta in cache", tbMeta->suid); + tfree(*pTableMeta); + *exist = 0; + return TSDB_CODE_SUCCESS; } - *exist = 1; + if ((*stbMeta)->suid != tbMeta->suid) { + CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + tfree(*pTableMeta); + ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); + *pTableMeta = realloc(*pTableMeta, metaSize); + if (NULL == *pTableMeta) { + CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + ctgError("calloc size[%d] failed", metaSize); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + + CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); return TSDB_CODE_SUCCESS; } @@ -225,9 +234,11 @@ int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray** vgroupList) { SHashObj *vgroupHash = NULL; SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + int32_t code = 0; - *vgroupList = taosArrayInit(taosHashGetSize(dbInfo->vgInfo), sizeof(SVgroupInfo)); - if (NULL == *vgroupList) { + vgList = taosArrayInit(taosHashGetSize(dbInfo->vgInfo), sizeof(SVgroupInfo)); + if (NULL == vgList) { ctgError("taosArrayInit failed"); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -236,29 +247,44 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * while (pIter) { vgInfo = pIter; - if (NULL == taosArrayPush(*vgroupList, vgInfo)) { + if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed"); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } pIter = taosHashIterate(dbInfo->vgInfo, pIter); vgInfo = NULL; } + *vgroupList = vgList; + vgList = NULL; + return TSDB_CODE_SUCCESS; + +_return: + + if (vgList) { + taosArrayDestroy(vgList); + } + + CTG_RET(code); } int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + int32_t code = 0; + + CTG_LOCK(CTG_READ, &dbInfo->lock); + int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); if (vgNum <= 0) { ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); } tableNameHashFp fp = NULL; SVgroupInfo *vgInfo = NULL; - CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); char tbFullName[TSDB_TABLE_FNAME_LEN]; @@ -279,19 +305,23 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, co if (NULL == vgInfo) { ctgError("no hash range found for hashvalue[%u]", hashValue); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } *pVgroup = *vgInfo; - return TSDB_CODE_SUCCESS; +_return: + + CTG_UNLOCK(CTG_READ, &dbInfo->lock); + + CTG_RET(TSDB_CODE_SUCCESS); } int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - + int32_t exist = 0; if (!forceUpdate) { @@ -316,21 +346,23 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { + int32_t code = 0; + if (output->metaNum != 1 && output->metaNum != 2) { ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == output->tbMeta) { ctgError("no valid table meta got from meta rsp"); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == pCatalog->tableCache.cache) { pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -338,50 +370,59 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.stableCache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } if (output->metaNum == 2) { if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { ctgError("push ctable[%s] to table cache failed", output->ctbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } if (TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } } int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - ctgError("push table[%s] to table cache failed", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { - if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) { + CTG_LOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { + CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + ctgError("push table[%s] to table cache failed", output->tbFname); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname)); + if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES) != 0) { + CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + } else { + if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { + ctgError("push table[%s] to table cache failed", output->tbFname); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } + +_return: + tfree(output->tbMeta); - return TSDB_CODE_SUCCESS; + CTG_RET(code); } -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { - if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t exist = 0; - +int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) { + bool inCache = false; if (0 == forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); - if (exist) { + if (inCache) { return TSDB_CODE_SUCCESS; } } @@ -397,9 +438,7 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup)); - if (dbInfo) { - *dbInfo = DbOut.dbVgroup; - } + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); return TSDB_CODE_SUCCESS; } @@ -479,17 +518,68 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, return TSDB_CODE_SUCCESS; } - SDBVgroupInfo * dbInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + SDBVgroupInfo * dbInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); if (NULL == dbInfo) { *version = CTG_DEFAULT_INVALID_VERSION; return TSDB_CODE_SUCCESS; } *version = dbInfo->vgVersion; + taosHashRelease(pCatalog->dbCache.cache, dbInfo); return TSDB_CODE_SUCCESS; } +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SArray** vgroupList) { + if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps || NULL == vgroupList) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + SDBVgroupInfo* db = NULL; + int32_t code = 0; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; + + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbName, forceUpdate, &db)); + + vgList = taosArrayInit(taosHashGetSize(db->vgInfo), sizeof(SVgroupInfo)); + if (NULL == vgList) { + ctgError("taosArrayInit failed"); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + void *pIter = taosHashIterate(db->vgInfo, NULL); + while (pIter) { + vgInfo = pIter; + + if (NULL == taosArrayPush(vgList, vgInfo)) { + ctgError("taosArrayPush failed"); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + pIter = taosHashIterate(db->vgInfo, pIter); + vgInfo = NULL; + } + + *vgroupList = vgList; + vgList = NULL; + +_return: + + if (db) { + CTG_UNLOCK(CTG_READ, &db->lock); + taosHashRelease(pCatalog->dbCache.cache, db); + } + + if (vgList) { + taosArrayDestroy(vgList); + vgList = NULL; + } + + CTG_RET(code); +} + + int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); @@ -497,13 +587,17 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB if (dbInfo->vgVersion < 0) { if (pCatalog->dbCache.cache) { - SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo && oldInfo->vgInfo) { - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; + SDBVgroupInfo *oldInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (oldInfo) { + CTG_LOCK(CTG_WRITE, &oldInfo->lock); + if (oldInfo->vgInfo) { + taosHashCleanup(oldInfo->vgInfo); + oldInfo->vgInfo = NULL; + } + CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); + + taosHashRelease(pCatalog->dbCache.cache, oldInfo); } - - taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName)); } ctgWarn("remove db [%s] from cache", dbName); @@ -517,10 +611,16 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } else { - SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo && oldInfo->vgInfo) { - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; + SDBVgroupInfo *oldInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (oldInfo) { + CTG_LOCK(CTG_WRITE, &oldInfo->lock); + if (oldInfo->vgInfo) { + taosHashCleanup(oldInfo->vgInfo); + oldInfo->vgInfo = NULL; + } + CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); + + taosHashRelease(pCatalog->dbCache.cache, oldInfo); } } @@ -573,60 +673,71 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo* dbVgroup = NULL; + SArray *vgList = NULL; + + *pVgroupList = NULL; CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta)); CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList)); + CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbVgroup, pVgroupList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(dbVgroup->vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("vgId[%d] not found in vgroup list", vgId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - *pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo)); - if (NULL == *pVgroupList) { + vgList = taosArrayInit(1, sizeof(SVgroupInfo)); + if (NULL == vgList) { ctgError("taosArrayInit failed"); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL == taosArrayPush(*pVgroupList, &vgroupInfo)) { + if (NULL == taosArrayPush(vgList, &vgroupInfo)) { ctgError("push vgroupInfo to array failed"); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } + + *pVgroupList = vgList; + vgList = NULL; } - tfree(tbMeta); - - return TSDB_CODE_SUCCESS; - _return: tfree(tbMeta); - taosArrayDestroy(*pVgroupList); - *pVgroupList = NULL; + if (dbVgroup) { + CTG_UNLOCK(CTG_READ, &dbVgroup->lock); + taosHashRelease(pCatalog->dbCache.cache, dbVgroup); + } + + if (vgList) { + taosArrayDestroy(vgList); + vgList = NULL; + } CTG_RET(code); } int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo dbInfo = {0}; + SDBVgroupInfo* dbInfo = NULL; int32_t code = 0; int32_t vgId = 0; CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, pDBName, false, &dbInfo)); - if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) { - ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo); - CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(dbInfo, pDBName, pTableName, pVgroup)); - CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup)); +_return: + + if (dbInfo) { + CTG_UNLOCK(CTG_READ, &dbInfo->lock); + taosHashRelease(pCatalog->dbCache.cache, dbInfo); + } CTG_RET(code); } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 07ca91729d..91927c370a 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -106,11 +106,11 @@ typedef struct SQWorkerMgmt { if (QW_READ == (type)) { \ if ((*(_lock)) < 0) assert(0); \ taosRLockLatch(_lock); \ - qDebug("RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ } else { \ if ((*(_lock)) < 0) assert(0); \ taosWLockLatch(_lock); \ - qDebug("WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ } \ } while (0) @@ -118,11 +118,11 @@ typedef struct SQWorkerMgmt { if (QW_READ == (type)) { \ if ((*(_lock)) <= 0) assert(0); \ taosRUnLockLatch(_lock); \ - qDebug("RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ } else { \ if ((*(_lock)) <= 0) assert(0); \ taosWUnLockLatch(_lock); \ - qDebug("WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ } \ } while (0) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index d343945a80..bf1774b45b 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -12,4 +12,6 @@ target_link_libraries( PUBLIC zlib PUBLIC lz4_static PUBLIC api -) \ No newline at end of file +) + +ADD_SUBDIRECTORY(test) diff --git a/source/util/src/thash.c b/source/util/src/thash.c index cfe14f00e1..840a1ef390 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -362,7 +362,7 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo return data; } -void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* d) { +void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void* d, bool acquire) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -404,6 +404,10 @@ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* memcpy(d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); } + if (acquire) { + pNode->count++; + } + data = GET_HASH_NODE_DATA(pNode); } @@ -415,6 +419,15 @@ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* return data; } +void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* d) { + return taosHashGetCloneImpl(pHashObj, key, keyLen, d, false); +} + +void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { + return taosHashGetCloneImpl(pHashObj, key, keyLen, NULL, true); +} + + int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, void *data, size_t dsize*/) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return -1; @@ -919,3 +932,9 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { __rd_unlock(&pHashObj->lock, pHashObj->type); } + +void taosHashRelease(SHashObj *pHashObj, void *p) { + taosHashCancelIterate(pHashObj, p); +} + + diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index a60c6cff28..79aaa1beb0 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -13,17 +13,22 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest tutil common os gtest pthread gcov) + TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread gcov) + LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/cacheTest.cpp) + LIST(APPEND SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/hashTest.cpp) + ADD_EXECUTABLE(hashTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(hashTest util common os gtest pthread gcov) + LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(trefTest ${BIN_SRC}) - TARGET_LINK_LIBRARIES(trefTest common tutil) + TARGET_LINK_LIBRARIES(trefTest common util) ENDIF() #IF (TD_LINUX) # ADD_EXECUTABLE(trefTest ./trefTest.c) -# TARGET_LINK_LIBRARIES(trefTest tutil common) +# TARGET_LINK_LIBRARIES(trefTest util common) #ENDIF () INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index bc3fed74c4..d31fcfb7ef 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -4,10 +4,15 @@ #include #include -#include "hash.h" +#include "thash.h" #include "taos.h" namespace { + +typedef struct TESTSTRUCT { + char *p; +}TESTSTRUCT; + // the simple test code for basic operations void simpleTest() { SHashObj* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); @@ -141,6 +146,52 @@ void invalidOperationTest() { } +void acquireRleaseTest() { + SHashObj* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + ASSERT_EQ(taosHashGetSize(hashTable), 0); + + int32_t key = 2; + int32_t code = 0; + int32_t num = 0; + TESTSTRUCT data = {0}; + char *str1 = "abcdefg"; + char *str2 = "aaaaaaa"; + char *str3 = "123456789"; + + data.p = (char *)malloc(10); + strcpy(data.p, str1); + + code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); + ASSERT_EQ(code, 0); + + TESTSTRUCT* pdata = (TESTSTRUCT*)taosHashAcquire(hashTable, &key, sizeof(key)); + ASSERT_TRUE(pdata != nullptr); + ASSERT_TRUE(strcmp(pdata->p, str1) == 0); + + code = taosHashRemove(hashTable, &key, sizeof(key)); + ASSERT_EQ(code, 0); + ASSERT_TRUE(strcmp(pdata->p, str1) == 0); + + num = taosHashGetSize(hashTable); + ASSERT_EQ(num, 1); + + strcpy(pdata->p, str3); + + data.p = (char *)malloc(10); + strcpy(data.p, str2); + code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); + ASSERT_EQ(code, 0); + num = taosHashGetSize(hashTable); + ASSERT_EQ(num, 2); + + printf("%s,expect:%s", pdata->p, str3); + ASSERT_TRUE(strcmp(pdata->p, str3) == 0); + + taosHashRelease(hashTable, pdata); + num = taosHashGetSize(hashTable); + ASSERT_EQ(num, 1); +} + } int main(int argc, char** argv) { @@ -153,4 +204,5 @@ TEST(testCase, hashTest) { stringKeyTest(); noLockPerformanceTest(); multithreadsTest(); + acquireRleaseTest(); } From c116946740ef83076edf7a40f82007da523a3602 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 04:05:10 -0800 Subject: [PATCH 06/97] create bnode and snode --- include/common/tmsg.h | 9 +- include/common/tmsgdef.h | 20 + include/dnode/mnode/sdb/sdb.h | 27 +- include/util/taoserror.h | 9 +- source/dnode/mgmt/impl/src/dndBnode.c | 4 +- source/dnode/mgmt/impl/src/dndQnode.c | 4 +- source/dnode/mgmt/impl/src/dndSnode.c | 4 +- .../impl/inc/{mndBalance.h => mndBnode.h} | 10 +- source/dnode/mnode/impl/inc/mndDef.h | 21 + .../impl/{src/mndBalance.c => inc/mndQnode.h} | 19 +- source/dnode/mnode/impl/inc/mndSnode.h | 32 ++ source/dnode/mnode/impl/src/mndBnode.c | 446 ++++++++++++++++++ source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndQnode.c | 446 ++++++++++++++++++ source/dnode/mnode/impl/src/mndSnode.c | 446 ++++++++++++++++++ source/dnode/mnode/impl/src/mnode.c | 8 +- source/dnode/mnode/sdb/src/sdbHash.c | 9 +- source/util/src/terror.c | 9 +- 18 files changed, 1488 insertions(+), 39 deletions(-) rename source/dnode/mnode/impl/inc/{mndBalance.h => mndBnode.h} (81%) rename source/dnode/mnode/impl/{src/mndBalance.c => inc/mndQnode.h} (74%) create mode 100644 source/dnode/mnode/impl/inc/mndSnode.h create mode 100644 source/dnode/mnode/impl/src/mndBnode.c create mode 100644 source/dnode/mnode/impl/src/mndQnode.c create mode 100644 source/dnode/mnode/impl/src/mndSnode.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ba8bb7ca2f..a6b6286fe7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -71,6 +71,9 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_TABLE, TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, + TSDB_MGMT_TABLE_QNODE, + TSDB_MGMT_TABLE_SNODE, + TSDB_MGMT_TABLE_BNODE, TSDB_MGMT_TABLE_VGROUP, TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_MODULE, @@ -866,15 +869,15 @@ typedef struct { typedef struct { int32_t dnodeId; -} SCreateQnodeInMsg, SDropQnodeInMsg; +} SMCreateQnodeMsg, SMCreateQnodeMsg, SDCreateQnodeMsg, SDDropQnodeMsg; typedef struct { int32_t dnodeId; -} SCreateSnodeInMsg, SDropSnodeInMsg; +} SMCreateSnodeMsg, SMCreateSnodeMsg, SDCreateSnodeMsg, SDDropSnodeMsg; typedef struct { int32_t dnodeId; -} SCreateBnodeInMsg, SDropBnodeInMsg; +} SMCreateBnodeMsg, SMCreateBnodeMsg, SDCreateBnodeMsg, SDDropBnodeMsg; typedef struct { int32_t dnodeId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 76ee31d9c9..d6071ebcf3 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -70,6 +70,15 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE, "dnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_QNODE, "dnode-alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_QNODE, "dnode-drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_SNODE, "dnode-create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_SNODE, "dnode-alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_SNODE, "dnode-drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_BNODE, "dnode-alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) @@ -90,9 +99,20 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_DROP_USER, "mnode-drop-user", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DNODE, "mnode-create-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_DNODE, "mnode-config-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DNODE, "mnode-alter-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DNODE, "mnode-drop-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "mnode-create-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_MNODE, "mnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "mnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_QNODE, "mnode-create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "mnode-alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "mnode-drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "mnode-create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "mnode-alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "mnode-drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "mnode-create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "mnode-alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "mnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "mnode-create-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "mnode-drop-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "mnode-use-db", NULL, NULL) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index f94090d7de..33f9dc5a1a 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -157,18 +157,21 @@ typedef enum { SDB_TRANS = 1, SDB_CLUSTER = 2, SDB_MNODE = 3, - SDB_DNODE = 4, - SDB_USER = 5, - SDB_AUTH = 6, - SDB_ACCT = 7, - SDB_CONSUMER = 8, - SDB_CGROUP = 9, - SDB_TOPIC = 10, - SDB_VGROUP = 11, - SDB_STB = 12, - SDB_DB = 13, - SDB_FUNC = 14, - SDB_MAX = 15 + SDB_QNODE = 4, + SDB_SNODE = 5, + SDB_BNODE = 6, + SDB_DNODE = 7, + SDB_USER = 8, + SDB_AUTH = 9, + SDB_ACCT = 10, + SDB_CONSUMER = 11, + SDB_CGROUP = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ba3e122db3..ae36ac7216 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -172,10 +172,15 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0347) #define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0348) -// mnode-mnode +// mnode-node #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) -#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0352) +#define TSDB_CODE_MND_QNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0352) +#define TSDB_CODE_MND_QNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0353) +#define TSDB_CODE_MND_SNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0354) +#define TSDB_CODE_MND_SNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0355) +#define TSDB_CODE_MND_BNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0356) +#define TSDB_CODE_MND_BNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0357) // mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 992f6ac0a1..66b619318d 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -256,7 +256,7 @@ static int32_t dndDropBnode(SDnode *pDnode) { } int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateBnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateBnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -268,7 +268,7 @@ int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropBnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropBnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 5d04a4f449..1f3d6ee371 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -261,7 +261,7 @@ static int32_t dndDropQnode(SDnode *pDnode) { } int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateQnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateQnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -273,7 +273,7 @@ int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropQnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropQnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 151fc7e6a1..ab4ca191a9 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -256,7 +256,7 @@ static int32_t dndDropSnode(SDnode *pDnode) { } int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateSnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateSnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -268,7 +268,7 @@ int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropSnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropSnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mnode/impl/inc/mndBalance.h b/source/dnode/mnode/impl/inc/mndBnode.h similarity index 81% rename from source/dnode/mnode/impl/inc/mndBalance.h rename to source/dnode/mnode/impl/inc/mndBnode.h index fa3679d033..cefc0b1859 100644 --- a/source/dnode/mnode/impl/inc/mndBalance.h +++ b/source/dnode/mnode/impl/inc/mndBnode.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_BALANCE_H_ -#define _TD_MND_BALANCE_H_ +#ifndef _TD_MND_BNODE_H_ +#define _TD_MND_BNODE_H_ #include "mndInt.h" @@ -22,11 +22,11 @@ extern "C" { #endif -int32_t mndInitBalance(SMnode *pMnode); -void mndCleanupBalance(SMnode *pMnode); +int32_t mndInitBnode(SMnode *pMnode); +void mndCleanupBnode(SMnode *pMnode); #ifdef __cplusplus } #endif -#endif /*_TD_MND_BALANCE_H_*/ +#endif /*_TD_MND_BNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a874e67210..b93fc0951c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -144,6 +144,27 @@ typedef struct { SDnodeObj *pDnode; } SMnodeObj; +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SQnodeObj; + +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SSnodeObj; + +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SBnodeObj; + typedef struct { int32_t maxUsers; int32_t maxDbs; diff --git a/source/dnode/mnode/impl/src/mndBalance.c b/source/dnode/mnode/impl/inc/mndQnode.h similarity index 74% rename from source/dnode/mnode/impl/src/mndBalance.c rename to source/dnode/mnode/impl/inc/mndQnode.h index 6ce89b96a3..5d177b3f6d 100644 --- a/source/dnode/mnode/impl/src/mndBalance.c +++ b/source/dnode/mnode/impl/inc/mndQnode.h @@ -13,9 +13,20 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#include "os.h" +#ifndef _TD_MND_QNODE_H_ +#define _TD_MND_QNODE_H_ + #include "mndInt.h" -int32_t mndInitBalance(SMnode *pMnode) { return 0; } -void mndCleanupBalance(SMnode *pMnode) {} \ No newline at end of file +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitQnode(SMnode *pMnode); +void mndCleanupQnode(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_QNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndSnode.h b/source/dnode/mnode/impl/inc/mndSnode.h new file mode 100644 index 0000000000..8d64879605 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndSnode.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_SNODE_H_ +#define _TD_MND_SNODE_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitSnode(SMnode *pMnode); +void mndCleanupSnode(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_SNODE_H_*/ diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c new file mode 100644 index 0000000000..e8a246e4d3 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndBnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_BNODE_VER_NUMBER 1 +#define TSDB_BNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj); +static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); +static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode); +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); + +int32_t mndInitBnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_BNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, + .insertFp = (SdbInsertFp)mndBnodeActionInsert, + .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndBnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndProcessCreateBnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndProcessDropBnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndGetBnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupBnode(SMnode *pMnode) {} + +static SBnodeObj *mndAcquireBnode(SMnode *pMnode, int32_t snodeId) { + SSdb *pSdb = pMnode->pSdb; + SBnodeObj *pObj = sdbAcquire(pSdb, SDB_BNODE, &snodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseBnode(SMnode *pMnode, SBnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_BNODE, TSDB_BNODE_VER_NUMBER, sizeof(SBnodeObj) + TSDB_BNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_BNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_BNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode snode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SBnodeObj)); + SBnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_BNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj) { + mTrace("snode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("snode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { + mTrace("snode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode) { + mTrace("snode:%d, perform update action", pOldBnode->id); + pOldBnode->updateTime = pNewBnode->updateTime; + return 0; +} + +static int32_t mndSetCreateBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDCreateBnodeMsg *pMsg = malloc(sizeof(SDCreateBnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pMsg->dnodeId); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateBnodeMsg); + action.msgType = TDMT_DND_CREATE_BNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateBnodeMsg *pCreate) { + SBnodeObj snodeObj = {0}; + snodeObj.id = pDnode->id; + snodeObj.createdTime = taosGetTimestampMs(); + snodeObj.updateTime = snodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_BNODE_OVER; + } + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateBnodeRedoLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndSetCreateBnodeCommitLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + code = 0; + +CREATE_BNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateBnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("snode:%d, start to create", pCreate->dnodeId); + + SBnodeObj *pObj = mndAcquireBnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("snode:%d, snode already exist", pObj->id); + mndReleaseBnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("snode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateBnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDDropBnodeMsg *pMsg = malloc(sizeof(SDDropBnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pMsg->dnodeId); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropBnodeMsg); + action.msgType = TDMT_DND_DROP_BNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pMsg, SBnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_BNODE_OVER; + } + + mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + + if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + code = 0; + +DROP_BNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateBnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("snode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SBnodeObj *pObj = mndAcquireBnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("snode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropBnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_BNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SBnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_BNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1800fd8e83..04e77c0136 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -329,7 +329,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SCreateMnodeMsg *pCreate) { SMnodeObj mnodeObj = {0}; - mnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_MNODE); + mnodeObj.id = pDnode->id; mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.updateTime = mnodeObj.createdTime; @@ -562,7 +562,7 @@ static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg) { return -1; } - sdbRelease(pMnode->pSdb, pMnode); + sdbRelease(pMnode->pSdb, pObj); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c new file mode 100644 index 0000000000..6d2ff555b9 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndQnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_QNODE_VER_NUMBER 1 +#define TSDB_QNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj); +static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); +static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode); +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); + +int32_t mndInitQnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_QNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, + .insertFp = (SdbInsertFp)mndQnodeActionInsert, + .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndQnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_QNODE, mndProcessCreateQnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_QNODE, mndProcessDropQnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndProcessCreateQnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndProcessDropQnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndGetQnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndRetrieveQnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndCancelGetNextQnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupQnode(SMnode *pMnode) {} + +static SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { + SSdb *pSdb = pMnode->pSdb; + SQnodeObj *pObj = sdbAcquire(pSdb, SDB_QNODE, &qnodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_QNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_QNODE, TSDB_QNODE_VER_NUMBER, sizeof(SQnodeObj) + TSDB_QNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_QNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_QNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode qnode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SQnodeObj)); + SQnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_QNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) { + mTrace("qnode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { + mTrace("qnode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode) { + mTrace("qnode:%d, perform update action", pOldQnode->id); + pOldQnode->updateTime = pNewQnode->updateTime; + return 0; +} + +static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDCreateQnodeMsg *pMsg = malloc(sizeof(SDCreateQnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pMsg->dnodeId); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateQnodeMsg); + action.msgType = TDMT_DND_CREATE_QNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateQnodeMsg *pCreate) { + SQnodeObj qnodeObj = {0}; + qnodeObj.id = pDnode->id; + qnodeObj.createdTime = taosGetTimestampMs(); + qnodeObj.updateTime = qnodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_QNODE_OVER; + } + mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + code = 0; + +CREATE_QNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateQnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("qnode:%d, start to create", pCreate->dnodeId); + + SQnodeObj *pObj = mndAcquireQnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("qnode:%d, qnode already exist", pObj->id); + mndReleaseQnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("qnode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateQnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDDropQnodeMsg *pMsg = malloc(sizeof(SDDropQnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pMsg->dnodeId); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropQnodeMsg); + action.msgType = TDMT_DND_DROP_QNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pMsg, SQnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("qnode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_QNODE_OVER; + } + + mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); + + if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + code = 0; + +DROP_QNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateQnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("qnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("qnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SQnodeObj *pObj = mndAcquireQnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("qnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropQnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_QNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SQnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_QNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c new file mode 100644 index 0000000000..03017a9b48 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndSnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_SNODE_VER_NUMBER 1 +#define TSDB_SNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj); +static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); +static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode); +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); + +int32_t mndInitSnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, + .insertFp = (SdbInsertFp)mndSnodeActionInsert, + .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndSnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SNODE, mndProcessCreateSnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SNODE, mndProcessDropSnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndProcessCreateSnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndProcessDropSnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndGetSnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndRetrieveSnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndCancelGetNextSnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupSnode(SMnode *pMnode) {} + +static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { + SSdb *pSdb = pMnode->pSdb; + SSnodeObj *pObj = sdbAcquire(pSdb, SDB_SNODE, &snodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_SNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_SNODE, TSDB_SNODE_VER_NUMBER, sizeof(SSnodeObj) + TSDB_SNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_SNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_SNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode snode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SSnodeObj)); + SSnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_SNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj) { + mTrace("snode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("snode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj) { + mTrace("snode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode) { + mTrace("snode:%d, perform update action", pOldSnode->id); + pOldSnode->updateTime = pNewSnode->updateTime; + return 0; +} + +static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDCreateSnodeMsg *pMsg = malloc(sizeof(SDCreateSnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pMsg->dnodeId); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateSnodeMsg); + action.msgType = TDMT_DND_CREATE_SNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateSnodeMsg *pCreate) { + SSnodeObj snodeObj = {0}; + snodeObj.id = pDnode->id; + snodeObj.createdTime = taosGetTimestampMs(); + snodeObj.updateTime = snodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_SNODE_OVER; + } + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + code = 0; + +CREATE_SNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateSnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("snode:%d, start to create", pCreate->dnodeId); + + SSnodeObj *pObj = mndAcquireSnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("snode:%d, snode already exist", pObj->id); + mndReleaseSnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("snode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateSnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDDropSnodeMsg *pMsg = malloc(sizeof(SDDropSnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pMsg->dnodeId); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropSnodeMsg); + action.msgType = TDMT_DND_DROP_SNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pMsg, SSnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_SNODE_OVER; + } + + mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + + if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + code = 0; + +DROP_SNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateSnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("snode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SSnodeObj *pObj = mndAcquireSnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("snode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropSnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_SNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SSnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_SNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 84076f8c0a..e902e13a36 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -16,14 +16,16 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" #include "mndAuth.h" -#include "mndBalance.h" +#include "mndBnode.h" #include "mndCluster.h" #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" #include "mndProfile.h" +#include "mndQnode.h" #include "mndShow.h" +#include "mndSnode.h" #include "mndStb.h" #include "mndSync.h" #include "mndTelem.h" @@ -147,6 +149,9 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitSnode, mndCleanupSnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; @@ -162,7 +167,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; } if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-balance", mndInitBalance, mndCleanupBalance) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 78a90b9a7d..06c2563910 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -24,6 +24,12 @@ static const char *sdbTableName(ESdbType type) { return "cluster"; case SDB_MNODE: return "mnode"; + case SDB_QNODE: + return "qnode"; + case SDB_SNODE: + return "snode"; + case SDB_BNODE: + return "bnode"; case SDB_DNODE: return "dnode"; case SDB_USER: @@ -55,7 +61,8 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) { } else if (keyType == SDB_KEY_INT32) { mTrace("%s:%d, refCount:%d oper:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, oper); } else if (keyType == SDB_KEY_INT64) { - mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, oper); + mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, + oper); } else { } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a923c6653..9fa5b3198b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -182,10 +182,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") -// mnode-mnode +// mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_ALREADY_EXIST, "Qnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_NOT_EXIST, "Qnode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") From 4e424e19df9d6aa80ea5ac1943d8f934f41a817b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 29 Dec 2021 12:15:56 +0000 Subject: [PATCH 07/97] add more functions --- include/common/tmsg.h | 12 +++++ include/util/encode.h | 8 ++-- source/common/src/tmsg.c | 95 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 49d0f9c97c..7b69647b7f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -57,8 +57,14 @@ extern int tMsgDict[]; typedef uint16_t tmsg_t; /* ------------------------ ENCODE/DECODE FUNCTIONS AND MACROS ------------------------ */ +struct SMEListNode { + TD_SLIST_NODE(SMEListNode); + SEncoder coder; +}; + typedef struct SMsgEncoder { SEncoder coder; + TD_SLIST(SMEListNode) eStack; // encode stack } SMsgEncoder; struct SMDFreeListNode { @@ -66,8 +72,14 @@ struct SMDFreeListNode { char payload[]; }; +struct SMDListNode { + TD_SLIST_NODE(SMDListNode); + SDecoder coder; +}; + typedef struct SMsgDecoder { SDecoder coder; + TD_SLIST(SMDListNode) dStack; TD_SLIST(SMDFreeListNode) freeList; } SMsgDecoder; diff --git a/include/util/encode.h b/include/util/encode.h index 85be2b76c6..ee35791012 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -26,8 +26,8 @@ extern "C" { typedef struct { td_endian_t endian; uint8_t* data; - int64_t size; - int64_t pos; + int32_t size; + int32_t pos; } SEncoder, SDecoder; #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) @@ -62,7 +62,7 @@ typedef struct { #define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) /* ------------------------ FOR ENCODER ------------------------ */ -static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int64_t size) { +static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int32_t size) { pEncoder->endian = endian; pEncoder->data = data; pEncoder->size = (data) ? size : 0; @@ -266,7 +266,7 @@ static FORCE_INLINE int tEncodeCStr(SEncoder* pEncoder, const char* val) { } /* ------------------------ FOR DECODER ------------------------ */ -static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int64_t size) { +static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int32_t size) { ASSERT(!TD_IS_NULL(data)); pDecoder->endian = endian; pDecoder->data = data; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4cabd1bb34..ed96384219 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,9 +27,30 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +static int tmsgStartEncode(SMsgEncoder *pME); +static void tmsgEndEncode(SMsgEncoder *pME); +static int tmsgStartDecode(SMsgDecoder *pMD); +static void tmsgEndDecode(SMsgDecoder *pMD); + /* ------------------------ ENCODE/DECODE FUNCTIONS ------------------------ */ +void tmsgInitMsgEncoder(SMsgEncoder *pME, td_endian_t endian, uint8_t *data, int64_t size) { + tInitEncoder(&(pME->coder), endian, data, size); + TD_SLIST_INIT(&(pME->eStack)); +} + +void tmsgClearMsgEncoder(SMsgEncoder *pME) { + struct SMEListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pME->eStack)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pME->eStack)); + free(pNode); + } +} + void tmsgInitMsgDecoder(SMsgDecoder *pMD, td_endian_t endian, uint8_t *data, int64_t size) { tInitDecoder(&pMD->coder, endian, data, size); + TD_SLIST_INIT(&(pMD->dStack)); TD_SLIST_INIT(&(pMD->freeList)); } @@ -45,12 +66,24 @@ void tmsgClearMsgDecoder(SMsgDecoder *pMD) { /* ------------------------ MESSAGE ENCODE/DECODE ------------------------ */ int tmsgSVCreateTbReqEncode(SMsgEncoder *pCoder, SVCreateTbReq *pReq) { + tmsgStartEncode(pCoder); // TODO + + tmsgEndEncode(pCoder); return 0; } int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) { - // TODO + tmsgStartDecode(pCoder); + + // TODO: decode + + // Decode is not end + if (pCoder->coder.pos != pCoder->coder.size) { + // Continue decode + } + + tmsgEndDecode(pCoder); return 0; } @@ -147,4 +180,64 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } return buf; +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static int tmsgStartEncode(SMsgEncoder *pME) { + struct SMEListNode *pNode = (struct SMEListNode *)malloc(sizeof(*pNode)); + if (TD_IS_NULL(pNode)) return -1; + + pNode->coder = pME->coder; + TD_SLIST_PUSH(&(pME->eStack), pNode); + TD_CODER_MOVE_POS(&(pME->coder), sizeof(int32_t)); + + return 0; +} + +static void tmsgEndEncode(SMsgEncoder *pME) { + int32_t size; + struct SMEListNode *pNode; + + pNode = TD_SLIST_HEAD(&(pME->eStack)); + ASSERT(pNode); + TD_SLIST_POP(&(pME->eStack)); + + size = pME->coder.pos - pNode->coder.pos; + tEncodeI32(&(pNode->coder), size); + + free(pNode); +} + +static int tmsgStartDecode(SMsgDecoder *pMD) { + struct SMDListNode *pNode; + int32_t size; + + pNode = (struct SMDListNode *)malloc(sizeof(*pNode)); + if (pNode == NULL) return -1; + + tDecodeI32(&(pMD->coder), &size); + + pNode->coder = pMD->coder; + TD_SLIST_PUSH(&(pMD->dStack), pNode); + + pMD->coder.pos = 0; + pMD->coder.size = size - sizeof(int32_t); + pMD->coder.data = TD_CODER_CURRENT(&(pNode->coder)); + + return 0; +} + +static void tmsgEndDecode(SMsgDecoder *pMD) { + ASSERT(pMD->coder.pos == pMD->coder.size); + struct SMDListNode *pNode; + + pNode = TD_SLIST_HEAD(&(pMD->dStack)); + ASSERT(pNode); + TD_SLIST_POP(&(pMD->dStack)); + + pNode->coder.pos += pMD->coder.size; + + pMD->coder = pNode->coder; + + free(pNode); } \ No newline at end of file From 96c053c6cf6b022bdb52fe302f1f41b5e4b84d17 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 29 Dec 2021 12:27:41 +0000 Subject: [PATCH 08/97] more --- source/common/src/tmsg.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index ed96384219..b81143ee62 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -55,12 +55,23 @@ void tmsgInitMsgDecoder(SMsgDecoder *pMD, td_endian_t endian, uint8_t *data, int } void tmsgClearMsgDecoder(SMsgDecoder *pMD) { - struct SMDFreeListNode *pNode; - for (;;) { - pNode = TD_SLIST_HEAD(&(pMD->freeList)); - if (TD_IS_NULL(pNode)) break; - TD_SLIST_POP(&(pMD->freeList)); - free(pNode); + { + struct SMDFreeListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pMD->freeList)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pMD->freeList)); + free(pNode); + } + } + { + struct SMDListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pMD->dStack)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pMD->dStack)); + free(pNode); + } } } From efe872c65987bf41eb8adb47b501016ff4eda4e7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Dec 2021 20:28:52 +0800 Subject: [PATCH 09/97] refactor code --- .../index/inc/index_fst_counting_writer.h | 3 +- source/libs/index/inc/index_tfile.h | 5 +- source/libs/index/src/index.c | 140 +++++++---- source/libs/index/src/index_cache.c | 136 +++++------ source/libs/index/src/index_fst_automation.c | 99 ++++---- .../index/src/index_fst_counting_writer.c | 10 +- source/libs/index/src/index_tfile.c | 231 +++++++++--------- source/libs/index/test/fstTest.cc | 8 +- source/libs/index/test/indexTests.cc | 108 ++++---- 9 files changed, 372 insertions(+), 368 deletions(-) diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ba0bd52e10..1504a69a08 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -37,6 +37,7 @@ typedef struct WriterCtx { struct { int fd; bool readOnly; + char buf[256]; } file; struct { int32_t capa; @@ -53,7 +54,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off static int writeCtxDoFlush(WriterCtx* ctx); WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); -void writerCtxDestroy(WriterCtx* w); +void writerCtxDestroy(WriterCtx* w, bool remove); typedef uint32_t CheckSummer; diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index f97a3126c8..d8e69b5fde 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -107,6 +107,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName); +TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName); TFileReader* tfileReaderCreate(WriterCtx* ctx); void tfileReaderDestroy(TFileReader* reader); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); @@ -114,10 +115,10 @@ void tfileReaderRef(TFileReader* reader); void tfileReaderUnRef(TFileReader* reader); TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t type); -void tfileWriteClose(TFileWriter* tw); +void tfileWriterClose(TFileWriter* tw); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); void tfileWriterDestroy(TFileWriter* tw); -int tfileWriterPut(TFileWriter* tw, void* data); +int tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b78c4ff258..6d64bdbd65 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -25,7 +25,7 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 4 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; @@ -52,18 +52,17 @@ typedef struct SIdxColInfo { static pthread_once_t isInit = PTHREAD_ONCE_INIT; // static void indexInit(); - static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); static void indexInterResultsDestroy(SArray* results); static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); +static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { // pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { - return -1; - } + if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -99,9 +98,7 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { - indexCacheUnRef(*pCache); - } + if (*pCache) { indexCacheUnRef(*pCache); } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); @@ -147,9 +144,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { - return ret; - } + if (ret != 0) { return ret; } } #endif @@ -179,9 +174,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } + for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -238,9 +231,7 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { - return NULL; - } + if (p == NULL) { return NULL; } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -262,9 +253,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { - return NULL; - } + if (t == NULL) { return NULL; } t->suid = suid; t->operType = oper; @@ -317,7 +306,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex - STermValueType s; + STermValueType s = kTypeValue; if (0 == indexCacheSearch(cache, query, *result, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by other opera", term->colName); @@ -336,9 +325,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { - return; - } + if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -368,23 +355,43 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType return 0; } -int indexFlushCacheTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { - return -1; +static void indexMergeSameKey(SArray* result, TFileValue* tv) { + int32_t sz = result ? taosArrayGetSize(result) : 0; + if (sz > 0) { + // TODO(yihao): remove duplicate tableid + TFileValue* lv = taosArrayGetP(result, sz - 1); + if (strcmp(lv->colVal, tv->colVal) == 0) { + taosArrayAddAll(lv->tableId, tv->tableId); + tfileValueDestroy(tv); + } else { + taosArrayPush(result, &tv); + } + } else { + taosArrayPush(result, &tv); } +} +static void indexDestroyTempResult(SArray* result) { + int32_t sz = result ? taosArrayGetSize(result) : 0; + for (size_t i = 0; i < sz; i++) { + TFileValue* tv = taosArrayGetP(result, i); + tfileValueDestroy(tv); + } + taosArrayDestroy(result); +} +int indexFlushCacheTFile(SIndex* sIdx, void* cache) { + if (sIdx == NULL) { return -1; } indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); - // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); SArray* result = taosArrayInit(1024, sizeof(void*)); - bool cn = cacheIter->next(cacheIter); - bool tn = tfileIter->next(tfileIter); + bool cn = cacheIter ? cacheIter->next(cacheIter) : false; + bool tn = tfileIter ? tfileIter->next(tfileIter) : false; while (cn == true && tn == true) { IterateValue* cv = cacheIter->getValue(cacheIter); IterateValue* tv = tfileIter->getValue(tfileIter); @@ -395,7 +402,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); taosArrayAddAll(tfv->tableId, tv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); @@ -403,13 +410,15 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { } else if (comp < 0) { TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); - taosArrayPush(result, &tfv); + + indexMergeSameKey(result, tfv); // copy to final Result; cn = cacheIter->next(cacheIter); } else { TFileValue* tfv = tfileValueCreate(tv->colVal); - taosArrayPush(result, &tfv); taosArrayAddAll(tfv->tableId, tv->val); + + indexMergeSameKey(result, tfv); // copy to final result tn = tfileIter->next(tfileIter); } @@ -418,33 +427,24 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { IterateValue* cv = cacheIter->getValue(cacheIter); TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); cn = cacheIter->next(cacheIter); } while (tn == true) { IterateValue* tv = tfileIter->getValue(tfileIter); TFileValue* tfv = tfileValueCreate(tv->colVal); + if (tv->val == NULL) { + // HO + printf("NO...."); + } taosArrayAddAll(tfv->tableId, tv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); tn = tfileIter->next(tfileIter); } - - int32_t version = CACHE_VERSION(pCache); - uint8_t colType = pCache->type; - - TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, pCache->colName, colType); - if (tw == NULL) { - indexError("faile to open file to write"); - } else { - int ret = tfileWriterPut(tw, result); - if (ret != 0) { - indexError("faile to write into tindex "); - } - } - // not free later, just put int table cache + int ret = indexGenTFile(sIdx, pCache, result); + indexDestroyTempResult(result); indexCacheDestroyImm(pCache); - tfileWriteClose(tw); indexCacheIteratorDestroy(cacheIter); tfileIteratorDestroy(tfileIter); @@ -455,9 +455,45 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { void iterateValueDestroy(IterateValue* value, bool destroy) { if (destroy) { taosArrayDestroy(value->val); + value->val = NULL; } else { - taosArrayClear(value->val); + if (value->val != NULL) { taosArrayClear(value->val); } } - free(value->colVal); + // free(value->colVal); value->colVal = NULL; } +static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { + int32_t version = CACHE_VERSION(cache); + uint8_t colType = cache->type; + + TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, cache->colName, colType); + if (tw == NULL) { + indexError("failed to open file to write"); + return -1; + } + + int ret = tfileWriterPut(tw, batch, true); + if (ret != 0) { + indexError("failed to write into tindex "); + goto END; + } + tfileWriterClose(tw); + + TFileReader* reader = tfileReaderOpen(sIdx->path, sIdx->suid, version, cache->colName); + + char buf[128] = {0}; + TFileHeader* header = &reader->header; + TFileCacheKey key = {.suid = header->suid, + .colName = header->colName, + .nColName = strlen(header->colName), + .colType = header->colType}; + pthread_mutex_lock(&sIdx->mtx); + + IndexTFile* ifile = (IndexTFile*)sIdx->tindex; + tfileCachePut(ifile->cache, &key, reader); + + pthread_mutex_unlock(&sIdx->mtx); + return ret; +END: + tfileWriterClose(tw); +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 217545d23b..9807969fd1 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 1000000 +#define MEM_TERM_LIMIT 100 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -86,25 +86,26 @@ void indexCacheDestroySkiplist(SSkipList* slt) { while (tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - if (ct != NULL) { - } + if (ct != NULL) {} } tSkipListDestroyIter(iter); tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { + if (cache == NULL) { return; } + MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread pthread_mutex_unlock(&cache->mtx); + + indexMemUnRef(tbl); indexMemUnRef(tbl); } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { - return; - } + if (pCache == NULL) { return; } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -114,9 +115,11 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { - return NULL; - } + if (iiter == NULL) { return NULL; } + + pthread_mutex_lock(&cache->mtx); + + indexMemRef(cache->imm); MemTable* tbl = cache->imm; iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); @@ -124,12 +127,12 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; + pthread_mutex_unlock(&cache->mtx); + return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -155,6 +158,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { taosMsleep(50); pthread_mutex_lock(&cache->mtx); } else { + indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); cache->nTerm = 1; @@ -166,17 +170,13 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -205,32 +205,11 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u IndexCache* pCache = cache; return 0; } -int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { - return -1; - } - IndexCache* pCache = cache; - SIndexTerm* term = query->term; - EIndexQueryType qtype = query->qType; - - MemTable *mem = NULL, *imm = NULL; - pthread_mutex_lock(&pCache->mtx); - mem = pCache->mem; - imm = pCache->imm; - indexMemRef(mem); - indexMemRef(imm); - pthread_mutex_unlock(&pCache->mtx); - - CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (ct == NULL) { - return -1; - } - ct->colVal = calloc(1, sizeof(char) * (term->nColVal + 1)); - memcpy(ct->colVal, term->colVal, term->nColVal); - ct->version = atomic_load_32(&pCache->version); +static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { + if (mem == NULL) { return 0; } char* key = getIndexKey(ct); - // TODO handle multi situation later, and refactor + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); @@ -251,51 +230,55 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } } tSkipListDestroyIter(iter); - cacheTermDestroy(ct); - // int32_t keyLen = CACHE_KEY_LEN(term); - // char* buf = calloc(1, keyLen); - if (qtype == QUERY_TERM) { - // - } else if (qtype == QUERY_PREFIX) { - // - } else if (qtype == QUERY_SUFFIX) { - // - } else if (qtype == QUERY_REGEX) { - // + return 0; +} +int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { + if (cache == NULL) { return -1; } + IndexCache* pCache = cache; + + MemTable *mem = NULL, *imm = NULL; + pthread_mutex_lock(&pCache->mtx); + mem = pCache->mem; + imm = pCache->imm; + indexMemRef(mem); + indexMemRef(imm); + pthread_mutex_unlock(&pCache->mtx); + + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + + int ret = indexQueryMem(mem, &ct, qtype, result, s); + if (ret == 0 && *s != kTypeDeletion) { + // continue search in imm + ret = indexQueryMem(imm, &ct, qtype, result, s); } + // cacheTermDestroy(ct); + indexMemUnRef(mem); indexMemUnRef(imm); - return 0; + + return ret; } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_DEC(cache); - if (ref == 0) { - indexCacheDestroy(cache); - } + if (ref == 0) { indexCacheDestroy(cache); } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -305,9 +288,7 @@ void indexMemUnRef(MemTable* tbl) { } static void cacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { - return; - } + if (ct == NULL) { return; } free(ct->colVal); free(ct); } @@ -322,9 +303,7 @@ static int32_t compareKey(const void* l, const void* r) { // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { - return rt->version - lt->version; - } + if (cmp == 0) { return rt->version - lt->version; } return cmp; } @@ -344,9 +323,7 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { - return false; - } + if (iter == NULL) { return false; } IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); @@ -356,7 +333,8 @@ static bool indexCacheIteratorNext(Iterate* itera) { CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); iv->type = ct->operaType; - iv->colVal = ct->colVal; + iv->colVal = calloc(1, strlen(ct->colVal) + 1); + memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); taosArrayPush(iv->val, &ct->uid); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 72983809d1..590ff294bf 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -17,7 +17,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = kind; nsv->type = ty; @@ -35,7 +37,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; - if (sv == NULL) { return; } + if (sv == NULL) { + return; + } if (sv->type == FST_INT) { // @@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) { } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = sv->kind; nsv->type = sv->type; @@ -65,24 +71,12 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { } // iterate fst -static void* alwaysMatchStart(AutomationCtx* ctx) { - return NULL; -} -static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { - return true; -} -static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { - return true; -} -static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} -static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { - return NULL; -} -static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* alwaysMatchStart(AutomationCtx* ctx) { return NULL; } +static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { return true; } +static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { return true; } +static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } +static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } +static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { return NULL; } // prefix query, impl later static void* prefixStart(AutomationCtx* ctx) { @@ -97,17 +91,20 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val >= 0; } -static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} +static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { StartWithStateValue* ssv = (StartWithStateValue*)state; - if (ssv == NULL || ctx == NULL) { return NULL; } + if (ssv == NULL || ctx == NULL) { + return NULL; + } char* data = ctx->data; - if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { - int val = ssv->val + 1; + int val = ssv->val + 1; + StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; @@ -118,35 +115,22 @@ static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { } return NULL; } -static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } // pattern query, impl later -static void* patternStart(AutomationCtx* ctx) { - return NULL; -} -static bool patternIsMatch(AutomationCtx* ctx, void* data) { - return true; -} -static bool patternCanMatch(AutomationCtx* ctx, void* data) { - return true; -} -static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} +static void* patternStart(AutomationCtx* ctx) { return NULL; } +static bool patternIsMatch(AutomationCtx* ctx, void* data) { return true; } +static bool patternCanMatch(AutomationCtx* ctx, void* data) { return true; } +static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } -static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { - return NULL; -} +static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } -static void* patternAcceptEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* patternAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } AutomationFunc automFuncs[] = { - {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, alwaysMatchAccpetEof}, + {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, + alwaysMatchAccpetEof}, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof} // add more search type @@ -154,7 +138,9 @@ AutomationFunc automFuncs[] = { AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { return NULL; } + if (ctx == NULL) { + return NULL; + } StartWithStateValue* sv = NULL; if (atype == AUTOMATION_ALWAYS) { @@ -170,11 +156,14 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { // add more search type } - char* src = (char*)data; - size_t len = strlen(src); - char* dst = (char*)malloc(len * sizeof(char) + 1); - memcpy(dst, src, len); - dst[len] = 0; + char* dst = NULL; + if (data != NULL) { + char* src = (char*)data; + size_t len = strlen(src); + dst = (char*)malloc(len * sizeof(char) + 1); + memcpy(dst, src, len); + dst[len] = 0; + } ctx->data = dst; ctx->type = atype; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index c8fbdd7690..fe2a5c767a 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -73,6 +73,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int } else { ctx->file.fd = tfOpenReadWrite(path); } + memcpy(ctx->file.buf, path, strlen(path)); if (ctx->file.fd < 0) { indexError("open file error %d", errno); goto END; @@ -95,11 +96,12 @@ END: free(ctx); return NULL; } -void writerCtxDestroy(WriterCtx* ctx) { +void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { tfClose(ctx->file.fd); + if (remove) unlink(ctx->file.buf); } free(ctx); } @@ -138,10 +140,8 @@ int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { - return 0; -} -int fstCountingWriterFlush(FstCountingWriter* write) { +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } +int fstCountingWriterFlush(FstCountingWriter* write) { WriterCtx* ctx = write->wrt; ctx->flush(ctx); // write->wtr->flush diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 271528a437..2d7b571a37 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -32,6 +32,7 @@ typedef struct TFileFstIter { #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) +static int tfileUidCompare(const void* a, const void* b); static int tfileStrCompare(const void* a, const void* b); static int tfileValueCompare(const void* a, const void* b, const void* param); static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); @@ -52,11 +53,16 @@ static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); +static TFileReader* tfileReaderCreateImpl(WriterCtx* ctx) { + TFileReader* reader = tfileReaderCreate(ctx); + tfileReaderRef(reader); + // tfileSerialCacheKey(&key, buf); + return reader; +} + TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { - return NULL; - } + if (tcache == NULL) { return NULL; } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; @@ -81,16 +87,13 @@ TFileCache* tfileCacheCreate(const char* path) { goto End; } - TFileReader* reader = tfileReaderCreate(wc); - tfileReaderRef(reader); - // loader fst and validate it + char buf[128] = {0}; + TFileReader* reader = tfileReaderCreateImpl(wc); TFileHeader* header = &reader->header; TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; - - char buf[128] = {0}; tfileSerialCacheKey(&key, buf); taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } @@ -102,9 +105,7 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { - return; - } + if (tcache == NULL) { return; } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -125,9 +126,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { tfileSerialCacheKey(key, buf); TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } tfileReaderRef(*reader); return *reader; @@ -137,7 +136,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) tfileSerialCacheKey(key, buf); // remove last version index reader TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); - if (*p != NULL) { + if (p != NULL) { TFileReader* oldReader = *p; taosHashRemove(tcache->tableCache, buf, strlen(buf)); tfileReaderUnRef(oldReader); @@ -149,9 +148,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } // T_REF_INC(reader); reader->ctx = ctx; @@ -172,12 +169,10 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { - return; - } + if (reader == NULL) { return; } // T_REF_INC(reader); fstDestroy(reader->fst); - writerCtxDestroy(reader->ctx); + writerCtxDestroy(reader->ctx, true); free(reader); } @@ -216,7 +211,8 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c char fullname[256] = {0}; snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); - WriterCtx* wcx = writerCtxCreate(TFile, fullname, true, 1024 * 1024); + WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024); + if (wcx == NULL) { return NULL; } TFileHeader tfh = {0}; tfh.suid = suid; @@ -226,6 +222,21 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c return tfileWriterCreate(wcx, &tfh); } +TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName) { + char filename[128] = {0}; + int32_t coldId = 1; + tfileGenFileName(filename, suid, coldId, version); + + char fullname[256] = {0}; + snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); + WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024); + if (wc == NULL) { return NULL; } + + TFileReader* reader = tfileReaderCreateImpl(wc); + return reader; + + // tfileSerialCacheKey(&key, buf); +} TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { // char pathBuf[128] = {0}; // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); @@ -249,28 +260,29 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { return tw; } -int tfileWriterPut(TFileWriter* tw, void* data) { +int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // sort by coltype and write to tindex - __compar_fn_t fn; - - int8_t colType = tw->header.colType; - if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { - fn = tfileStrCompare; - } else { - fn = getComparFunc(colType, 0); + if (order == false) { + __compar_fn_t fn; + int8_t colType = tw->header.colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + fn = tfileStrCompare; + } else { + fn = getComparFunc(colType, 0); + } + taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); } - taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); - int32_t bufLimit = 4096, offset = 0; - char* buf = calloc(1, sizeof(char) * bufLimit); - char* p = buf; + int32_t bufLimit = 64 * 4096, offset = 0; + // char* buf = calloc(1, sizeof(char) * bufLimit); + // char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; // ugly code, refactor later for (size_t i = 0; i < sz; i++) { TFileValue* v = taosArrayGetP((SArray*)data, i); - + // taosArrayRemoveDuplicate(v->tablId, tfileUidCompare, NULL); int32_t tbsz = taosArrayGetSize(v->tableId); fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); } @@ -282,30 +294,39 @@ int tfileWriterPut(TFileWriter* tw, void* data) { int32_t tbsz = taosArrayGetSize(v->tableId); // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - if (offset + ttsz > bufLimit) { - // batch write - tw->ctx->write(tw->ctx, buf, offset); - offset = 0; - memset(buf, 0, bufLimit); - p = buf; - } + // if (offset + ttsz >= bufLimit) { + // // batch write + // indexInfo("offset: %d, ttsz: %d", offset, ttsz); + // // std::cout << "offset: " << offset << std::endl; + // // std::cout << "ttsz:" << ttsz < < < std::endl; + // tw->ctx->write(tw->ctx, buf, offset); + // offset = 0; + // memset(buf, 0, bufLimit); + // p = buf; + //} + // if (ttsz >= bufLimit) { + //} + char* buf = calloc(1, ttsz * sizeof(char)); + char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); - offset += ttsz; - p = buf + offset; + tw->ctx->write(tw->ctx, buf, ttsz); + // offset += ttsz; + // p = buf + offset; // set up value offset v->offset = tw->offset; tw->offset += ttsz; + free(buf); } - if (offset != 0) { - // write reversed data in buf to tindex - tw->ctx->write(tw->ctx, buf, offset); - } - tfree(buf); + // if (offset != 0) { + // write reversed data in buf to tindex + // tw->ctx->write(tw->ctx, buf, offset); + //} + // tfree(buf); tw->fb = fstBuilderCreate(tw->ctx, 0); if (tw->fb == NULL) { - tfileWriterDestroy(tw); + tfileWriterClose(tw); return -1; } // write fst @@ -321,27 +342,20 @@ int tfileWriterPut(TFileWriter* tw, void* data) { tw->fb = NULL; return 0; } -void tfileWriteClose(TFileWriter* tw) { - if (tw == NULL) { - return; - } - writerCtxDestroy(tw->ctx); +void tfileWriterClose(TFileWriter* tw) { + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { - return; - } - - writerCtxDestroy(tw->ctx); + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); - if (tfile == NULL) { - return NULL; - } + if (tfile == NULL) { return NULL; } tfile->cache = tfileCacheCreate(path); return tfile; @@ -350,18 +364,14 @@ void IndexTFileDestroy(IndexTFile* tfile) { free(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { - return ret; - } + if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; TFileCacheKey key = { .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { - return 0; - } + if (reader == NULL) { return 0; } return tfileReaderSearch(reader, query, result); } @@ -381,9 +391,7 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { - return false; - } + if (rt == NULL) { return false; } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -391,12 +399,9 @@ static bool tfileIteratorNext(Iterate* iiter) { memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); - swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { - return false; - } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; @@ -407,9 +412,8 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(Iterate)); - if (tIter == NULL) { - return NULL; - } + if (tIter == NULL) { return NULL; } + tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); tIter->st = streamBuilderIntoStream(tIter->fb); @@ -418,21 +422,22 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - Iterate* iter = calloc(1, sizeof(Iterate)); + if (reader == NULL) { return NULL; } + Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { + free(iter); return NULL; } - iter->next = tfileIteratorNext; iter->getValue = tifileIterateGetValue; + iter->val.val = taosArrayInit(1, sizeof(uint64_t)); return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } + IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -445,18 +450,19 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { - if (tf == NULL) { - return NULL; - } + if (tf == NULL) { return NULL; } TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } +static int tfileUidCompare(const void* a, const void* b) { + uint64_t l = *(uint64_t*)a; + uint64_t r = *(uint64_t*)b; + return l - r; +} static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } @@ -471,17 +477,13 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { - return NULL; - } - + if (tf == NULL) { return NULL; } + tf->colVal = val; tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { - return -1; - } + if (tf == NULL) { return -1; } taosArrayPush(tf->tableId, &val); return 0; } @@ -501,9 +503,7 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { - return -1; - } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } tw->offset += sizeof(fstOffset); return 0; } @@ -514,9 +514,7 @@ static int tfileWriteHeader(TFileWriter* writer) { memcpy(buf, (char*)header, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { - return -1; - } + if (sizeof(buf) != nwrite) { return -1; } writer->offset = nwrite; return 0; } @@ -547,12 +545,10 @@ static int tfileReaderLoadHeader(TFileReader* reader) { } static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later - static int FST_MAX_SIZE = 16 * 1024; + static int FST_MAX_SIZE = 64 * 1024; char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); @@ -575,36 +571,33 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* int32_t total = sizeof(uint64_t) * nid; char* buf = calloc(1, total); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } nread = ctx->read(ctx, buf, total); assert(total == nread); - for (int32_t i = 0; i < nid; i++) { - taosArrayPush(result, (uint64_t*)buf + i); - } + for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } free(buf); return 0; } void tfileReaderRef(TFileReader* reader) { + if (reader == NULL) { return; } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { + if (reader == NULL) { return; } int ref = T_REF_DEC(reader); if (ref == 0) { + // do nothing tfileReaderDestroy(reader); } } static int tfileGetFileList(const char* path, SArray* result) { DIR* dir = opendir(path); - if (NULL == dir) { - return -1; - } + if (NULL == dir) { return -1; } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { @@ -632,9 +625,7 @@ static int tfileCompare(const void* a, const void* b) { size_t bLen = strlen(bName); int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } // tfile name suid-colId-version.tindex diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 85bb5e2b15..a4d8bb36f1 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -12,9 +12,7 @@ #include "tskiplist.h" #include "tutil.h" -void* callback(void* s) { - return s; -} +void* callback(void* s) { return s; } static std::string fileName = "/tmp/tindex.tindex"; class FstWriter { @@ -34,7 +32,7 @@ class FstWriter { fstBuilderFinish(_b); fstBuilderDestroy(_b); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: @@ -102,7 +100,7 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 110499f67a..3d06dde98f 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -57,7 +57,7 @@ class FstWriter { fstBuilderFinish(_b); fstBuilderDestroy(_b); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: @@ -76,9 +76,7 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { - return false; - } + if (nRead <= 0) { return false; } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -104,9 +102,7 @@ class FstReadMemory { StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; - while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { - result.push_back((uint64_t)(rt->out.out)); - } + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { result.push_back((uint64_t)(rt->out.out)); } return true; } bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { @@ -120,7 +116,7 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, true); } private: @@ -182,9 +178,7 @@ void checkFstPerf() { delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); - if (m->init()) { - printf("success to init fst read"); - } + if (m->init()) { printf("success to init fst read"); } Performance_fstReadRecords(m); delete m; } @@ -346,10 +340,8 @@ class TFileObj { tfileReaderDestroy(reader_); reader_ = NULL; } - if (writer_ == NULL) { - InitWriter(); - } - return tfileWriterPut(writer_, tv); + if (writer_ == NULL) { InitWriter(); } + return tfileWriterPut(writer_, tv, false); } bool InitWriter() { TFileHeader header; @@ -388,12 +380,8 @@ class TFileObj { return tfileReaderSearch(reader_, query, result); } ~TFileObj() { - if (writer_) { - tfileWriterDestroy(writer_); - } - if (reader_) { - tfileReaderDestroy(reader_); - } + if (writer_) { tfileWriterDestroy(writer_); } + if (reader_) { tfileReaderDestroy(reader_); } } private: @@ -465,9 +453,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { taosArrayPush(data, &v4); fObj->Put(data); - for (size_t i = 0; i < taosArrayGetSize(data); i++) { - destroyTFileValue(taosArrayGetP(data, i)); - } + for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); } taosArrayDestroy(data); std::string colName("voltage"); @@ -569,7 +555,8 @@ TEST_F(IndexCacheEnv, cache_test) { colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); } - + coj->Debug(); + std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), @@ -582,6 +569,8 @@ TEST_F(IndexCacheEnv, cache_test) { colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); } + coj->Debug(); + std::cout << "--------second----------" << std::endl; { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { @@ -602,7 +591,7 @@ TEST_F(IndexCacheEnv, cache_test) { STermValueType valType; coj->Get(&query, colId, 10000, ret, &valType); - // std::cout << "size : " << taosArrayGetSize(ret) << std::endl; + std::cout << "size : " << taosArrayGetSize(ret) << std::endl; assert(taosArrayGetSize(ret) == 4); } { @@ -635,6 +624,20 @@ class IndexObj { } return ret; } + int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", + size_t numOfTable = 100 * 10000) { + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < numOfTable; i++) { + int ret = Put(terms, i); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + return numOfTable; + } + int Put(SIndexMultiTerm* fvs, uint64_t uid) { numOfWrite += taosArrayGetSize(fvs); return indexPut(idx, fvs, uid); @@ -645,6 +648,17 @@ class IndexObj { return indexSearch(idx, multiQ, result); } + int SearchOne(const std::string& colName, const std::string& colVal) { + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + indexMultiTermQueryAdd(mq, term, QUERY_TERM); + + SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); + if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; } + return taosArrayGetSize(result); + // assert(taosArrayGetSize(result) == targetSize); + } void Debug() { std::cout << "numOfWrite:" << numOfWrite << std::endl; std::cout << "numOfRead:" << numOfRead << std::endl; @@ -665,12 +679,13 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: virtual void SetUp() { + tfInit(); index = new IndexObj(); // } virtual void TearDown() { - // r delete index; + tfCleanup(); } IndexObj* index; }; @@ -722,44 +737,39 @@ TEST_F(IndexEnv2, testIndexOpen) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); index->Search(mq, result); - assert(taosArrayGetSize(result) == targetSize); + std::cout << "target size: " << taosArrayGetSize(result) << std::endl; + // assert(taosArrayGetSize(result) == targetSize); } } TEST_F(IndexEnv2, testIndex_TrigeFlush) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} + int numOfTable = 100 * 10000; + index->WriteMillonData("tag1", "Hello world", numOfTable); + int target = index->SearchOne("tag1", "Hellow world"); + assert(numOfTable == target); } -TEST_F(IndexEnv2, testIndex_TrigeFlush) { +TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } -TEST_(IndexEnv2, testIndex_serarch_cache_and_tfile) { +TEST_F(IndexEnv2, testIndex_multi_thread_write) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } -TEST_(IndexEnv2, testIndex_multi_thread_write) { +TEST_F(IndexEnv2, testIndex_multi_thread_read) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } -} -TEST_(IndexEnv2, testIndex_multi_thread_read) { - std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } -TEST_(IndexEnv2, testIndex_restart) { +TEST_F(IndexEnv2, testIndex_restart) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } TEST_F(IndexEnv2, testIndex_performance) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } +TEST_F(IndexEnv2, testIndexMultiTag) {} From 736c90adfda8a173843e7db648d1f21fcb8ea3bb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 04:31:04 -0800 Subject: [PATCH 10/97] add qnode test --- include/common/tmsg.h | 6 +- source/dnode/mgmt/impl/src/dndDnode.c | 23 ++- source/dnode/mgmt/impl/src/dndTransport.c | 18 ++ source/dnode/mgmt/impl/test/CMakeLists.txt | 3 + .../dnode/mgmt/impl/test/qnode/CMakeLists.txt | 11 ++ source/dnode/mgmt/impl/test/qnode/qnode.cpp | 170 ++++++++++++++++++ source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndShow.c | 6 + source/dnode/mnode/impl/src/mndSnode.c | 4 +- source/dnode/qnode/src/qnode.c | 6 +- source/dnode/snode/src/snode.c | 2 +- 12 files changed, 241 insertions(+), 16 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/qnode/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/qnode/qnode.cpp diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a6b6286fe7..1a88cecf91 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -869,15 +869,15 @@ typedef struct { typedef struct { int32_t dnodeId; -} SMCreateQnodeMsg, SMCreateQnodeMsg, SDCreateQnodeMsg, SDDropQnodeMsg; +} SMCreateQnodeMsg, SMDropQnodeMsg, SDCreateQnodeMsg, SDDropQnodeMsg; typedef struct { int32_t dnodeId; -} SMCreateSnodeMsg, SMCreateSnodeMsg, SDCreateSnodeMsg, SDDropSnodeMsg; +} SMCreateSnodeMsg, SMDropSnodeMsg, SDCreateSnodeMsg, SDDropSnodeMsg; typedef struct { int32_t dnodeId; -} SMCreateBnodeMsg, SMCreateBnodeMsg, SDCreateBnodeMsg, SDDropBnodeMsg; +} SMCreateBnodeMsg, SMDropBnodeMsg, SDCreateBnodeMsg, SDDropBnodeMsg; typedef struct { int32_t dnodeId; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 98763fce30..8807579564 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -15,9 +15,12 @@ #define _DEFAULT_SOURCE #include "dndDnode.h" +#include "dndBnode.h" +#include "dndMnode.h" +#include "dndQnode.h" +#include "dndSnode.h" #include "dndTransport.h" #include "dndVnodes.h" -#include "dndMnode.h" static int32_t dndInitMgmtWorker(SDnode *pDnode); static void dndCleanupMgmtWorker(SDnode *pDnode); @@ -648,6 +651,24 @@ static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { case TDMT_DND_DROP_MNODE: code = dndProcessDropMnodeReq(pDnode, pMsg); break; + case TDMT_DND_CREATE_QNODE: + code = dndProcessCreateQnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_QNODE: + code = dndProcessDropQnodeReq(pDnode, pMsg); + break; + case TDMT_DND_CREATE_SNODE: + code = dndProcessCreateSnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_SNODE: + code = dndProcessDropSnodeReq(pDnode, pMsg); + break; + case TDMT_DND_CREATE_BNODE: + code = dndProcessCreateBnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_BNODE: + code = dndProcessDropBnodeReq(pDnode, pMsg); + break; case TDMT_DND_CONFIG_DNODE: code = dndProcessConfigDnodeReq(pDnode, pMsg); break; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8db09ea4cd..ba93d1913d 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -37,6 +37,18 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg; @@ -66,6 +78,12 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index dcce270d7d..dde2c36dbd 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -8,6 +8,9 @@ add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) +add_subdirectory(qnode) +#add_subdirectory(mnode) +#add_subdirectory(mnode) add_subdirectory(profile) add_subdirectory(show) add_subdirectory(stb) diff --git a/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt new file mode 100644 index 0000000000..2edb6de8b8 --- /dev/null +++ b/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. QTEST_SRC) +add_executable(dnode_test_qnode ${QTEST_SRC}) +target_link_libraries( + dnode_test_qnode + PUBLIC sut +) + +add_test( + NAME dnode_test_qnode + COMMAND dnode_test_qnode +) diff --git a/source/dnode/mgmt/impl/test/qnode/qnode.cpp b/source/dnode/mgmt/impl/test/qnode/qnode.cpp new file mode 100644 index 0000000000..2383bea177 --- /dev/null +++ b/source/dnode/mgmt/impl/test/qnode/qnode.cpp @@ -0,0 +1,170 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestQnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_qnode1", 9071); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9071"; + + server2.Start("/tmp/dnode_test_qnode2", fqdn, 9072, firstEp); + server3.Start("/tmp/dnode_test_qnode3", fqdn, 9073, firstEp); + server4.Start("/tmp/dnode_test_qnode4", fqdn, 9074, firstEp); + server5.Start("/tmp/dnode_test_qnode5", fqdn, 9075, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + server3.Stop(); + server4.Stop(); + server5.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; + static TestServer server3; + static TestServer server4; + static TestServer server5; +}; + +Testbase DndTestQnode::test; +TestServer DndTestQnode::server2; +TestServer DndTestQnode::server3; +TestServer DndTestQnode::server4; +TestServer DndTestQnode::server5; + +// TEST_F(DndTestQnode, 01_ShowQnode) { +// test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); +// CHECK_META("show qnodes", 3); + +// CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); +// CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); +// CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + +// test.SendShowRetrieveMsg(); +// EXPECT_EQ(test.GetShowRows(), 1); + +// CheckInt16(1); +// CheckBinary("localhost:9071", TSDB_EP_LEN); +// CheckTimestamp(); +// } + +TEST_F(DndTestQnode, 02_Create_Qnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9071", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +// TEST_F(DndTestQnode, 03_Create_Qnode_Invalid_Id) { +// { +// int32_t contLen = sizeof(SMCreateQnodeMsg); + +// SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); +// pReq->dnodeId = htonl(2); + +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); +// ASSERT_NE(pMsg, nullptr); +// ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); +// } +// } + +// TEST_F(DndTestQnode, 04_Create_Qnode) { +// { +// // create dnode +// int32_t contLen = sizeof(SCreateDnodeMsg); + +// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); +// strcpy(pReq->fqdn, "localhost"); +// pReq->port = htonl(9072); + +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); +// ASSERT_NE(pMsg, nullptr); +// ASSERT_EQ(pMsg->code, 0); + +// taosMsleep(1300); +// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); +// test.SendShowRetrieveMsg(); +// EXPECT_EQ(test.GetShowRows(), 2); +// } + +// { +// // create qnode +// int32_t contLen = sizeof(SMCreateQnodeMsg); + +// SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); +// pReq->dnodeId = htonl(2); + +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); +// ASSERT_NE(pMsg, nullptr); +// ASSERT_EQ(pMsg->code, 0); + +// test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); +// test.SendShowRetrieveMsg(); +// EXPECT_EQ(test.GetShowRows(), 2); + +// CheckInt16(1); +// CheckInt16(2); +// CheckBinary("localhost:9071", TSDB_EP_LEN); +// CheckBinary("localhost:9072", TSDB_EP_LEN); +// CheckTimestamp(); +// CheckTimestamp(); +// } + +// { +// // drop qnode +// int32_t contLen = sizeof(SMDropQnodeMsg); + +// SMDropQnodeMsg* pReq = (SMDropQnodeMsg*)rpcMallocCont(contLen); +// pReq->dnodeId = htonl(2); + +// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); +// ASSERT_NE(pMsg, nullptr); +// ASSERT_EQ(pMsg->code, 0); + +// test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); +// test.SendShowRetrieveMsg(); +// EXPECT_EQ(test.GetShowRows(), 1); + +// CheckInt16(1); +// CheckBinary("localhost:9071", TSDB_EP_LEN); +// CheckTimestamp(); +// } +// } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index e8a246e4d3..6018277a13 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -324,8 +324,8 @@ DROP_BNODE_OVER: } static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMCreateBnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + SMDropBnodeMsg *pDrop = pMsg->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("snode:%d, start to drop", pDrop->dnodeId); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 6d2ff555b9..4afd20e2b6 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -324,8 +324,8 @@ DROP_QNODE_OVER: } static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMCreateQnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + SMDropQnodeMsg *pDrop = pMsg->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("qnode:%d, start to drop", pDrop->dnodeId); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index b89de94452..af633355ef 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -262,6 +262,12 @@ char *mndShowStr(int32_t showType) { return "show dnodes"; case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; + case TSDB_MGMT_TABLE_QNODE: + return "show qnodes"; + case TSDB_MGMT_TABLE_SNODE: + return "show snodes"; + case TSDB_MGMT_TABLE_BNODE: + return "show bnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; case TSDB_MGMT_TABLE_STB: diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 03017a9b48..083144c3c6 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -324,8 +324,8 @@ DROP_SNODE_OVER: } static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SMCreateSnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + SMDropSnodeMsg *pDrop = pMsg->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("snode:%d, start to drop", pDrop->dnodeId); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index ff1a460228..8d2a72b268 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -24,12 +24,8 @@ void qndClose(SQnode *pQnode) { free(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } -int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } -int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - *pRsp = NULL; - return 0; -} diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 7ae4d49059..5d21a037e7 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -24,7 +24,7 @@ void sndClose(SSnode *pSnode) { free(pSnode); } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } -int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } From 90055852b69df8510683b809842ad6b640ab92ae Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 04:42:52 -0800 Subject: [PATCH 11/97] unittest --- source/dnode/mgmt/impl/test/CMakeLists.txt | 4 +- .../dnode/mgmt/impl/test/bnode/CMakeLists.txt | 11 ++ source/dnode/mgmt/impl/test/bnode/bnode.cpp | 154 +++++++++++++++++ source/dnode/mgmt/impl/test/qnode/qnode.cpp | 160 ++++++++---------- .../dnode/mgmt/impl/test/snode/CMakeLists.txt | 11 ++ source/dnode/mgmt/impl/test/snode/snode.cpp | 154 +++++++++++++++++ source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- 9 files changed, 410 insertions(+), 96 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/bnode/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/bnode/bnode.cpp create mode 100644 source/dnode/mgmt/impl/test/snode/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/snode/snode.cpp diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index dde2c36dbd..b0596bed08 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -9,8 +9,8 @@ add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) add_subdirectory(qnode) -#add_subdirectory(mnode) -#add_subdirectory(mnode) +add_subdirectory(snode) +add_subdirectory(bnode) add_subdirectory(profile) add_subdirectory(show) add_subdirectory(stb) diff --git a/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt new file mode 100644 index 0000000000..ea769a701c --- /dev/null +++ b/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. STEST_SRC) +add_executable(dnode_test_bnode ${STEST_SRC}) +target_link_libraries( + dnode_test_bnode + PUBLIC sut +) + +add_test( + NAME dnode_test_bnode + COMMAND dnode_test_bnode +) diff --git a/source/dnode/mgmt/impl/test/bnode/bnode.cpp b/source/dnode/mgmt/impl/test/bnode/bnode.cpp new file mode 100644 index 0000000000..b1a85522c4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/bnode/bnode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestBnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_bnode1", 9068); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9068"; + + server2.Start("/tmp/dnode_test_bnode2", fqdn, 9069, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestBnode::test; +TestServer DndTestBnode::server2; + +TEST_F(DndTestBnode, 01_ShowBnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + CHECK_META("show bnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestBnode, 02_Create_Bnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + CHECK_META("show bnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestBnode, 03_Create_Bnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestBnode, 04_Create_Bnode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9069); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create bnode + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckBinary("localhost:9069", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop bnode + int32_t contLen = sizeof(SMDropBnodeMsg); + + SMDropBnodeMsg* pReq = (SMDropBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/qnode/qnode.cpp b/source/dnode/mgmt/impl/test/qnode/qnode.cpp index 2383bea177..40464de816 100644 --- a/source/dnode/mgmt/impl/test/qnode/qnode.cpp +++ b/source/dnode/mgmt/impl/test/qnode/qnode.cpp @@ -18,53 +18,37 @@ class DndTestQnode : public ::testing::Test { public: static void SetUpTestSuite() { - test.Init("/tmp/dnode_test_qnode1", 9071); + test.Init("/tmp/dnode_test_qnode1", 9064); const char* fqdn = "localhost"; - const char* firstEp = "localhost:9071"; + const char* firstEp = "localhost:9064"; - server2.Start("/tmp/dnode_test_qnode2", fqdn, 9072, firstEp); - server3.Start("/tmp/dnode_test_qnode3", fqdn, 9073, firstEp); - server4.Start("/tmp/dnode_test_qnode4", fqdn, 9074, firstEp); - server5.Start("/tmp/dnode_test_qnode5", fqdn, 9075, firstEp); + server2.Start("/tmp/dnode_test_qnode2", fqdn, 9065, firstEp); taosMsleep(300); } static void TearDownTestSuite() { server2.Stop(); - server3.Stop(); - server4.Stop(); - server5.Stop(); test.Cleanup(); } static Testbase test; static TestServer server2; - static TestServer server3; - static TestServer server4; - static TestServer server5; }; Testbase DndTestQnode::test; TestServer DndTestQnode::server2; -TestServer DndTestQnode::server3; -TestServer DndTestQnode::server4; -TestServer DndTestQnode::server5; -// TEST_F(DndTestQnode, 01_ShowQnode) { -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); -// CHECK_META("show qnodes", 3); +TEST_F(DndTestQnode, 01_ShowQnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); -// CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); -// CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); -// CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); -// test.SendShowRetrieveMsg(); -// EXPECT_EQ(test.GetShowRows(), 1); - -// CheckInt16(1); -// CheckBinary("localhost:9071", TSDB_EP_LEN); -// CheckTimestamp(); -// } + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} TEST_F(DndTestQnode, 02_Create_Qnode_Invalid_Id) { { @@ -88,83 +72,83 @@ TEST_F(DndTestQnode, 02_Create_Qnode_Invalid_Id) { EXPECT_EQ(test.GetShowRows(), 1); CheckInt16(1); - CheckBinary("localhost:9071", TSDB_EP_LEN); + CheckBinary("localhost:9064", TSDB_EP_LEN); CheckTimestamp(); } } -// TEST_F(DndTestQnode, 03_Create_Qnode_Invalid_Id) { -// { -// int32_t contLen = sizeof(SMCreateQnodeMsg); +TEST_F(DndTestQnode, 03_Create_Qnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateQnodeMsg); -// SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); -// pReq->dnodeId = htonl(2); + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); -// } -// } + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} -// TEST_F(DndTestQnode, 04_Create_Qnode) { -// { -// // create dnode -// int32_t contLen = sizeof(SCreateDnodeMsg); +TEST_F(DndTestQnode, 04_Create_Qnode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); -// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); -// strcpy(pReq->fqdn, "localhost"); -// pReq->port = htonl(9072); + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9065); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); -// taosMsleep(1300); -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); -// test.SendShowRetrieveMsg(); -// EXPECT_EQ(test.GetShowRows(), 2); -// } + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } -// { -// // create qnode -// int32_t contLen = sizeof(SMCreateQnodeMsg); + { + // create qnode + int32_t contLen = sizeof(SMCreateQnodeMsg); -// SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); -// pReq->dnodeId = htonl(2); + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); -// test.SendShowRetrieveMsg(); -// EXPECT_EQ(test.GetShowRows(), 2); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); -// CheckInt16(1); -// CheckInt16(2); -// CheckBinary("localhost:9071", TSDB_EP_LEN); -// CheckBinary("localhost:9072", TSDB_EP_LEN); -// CheckTimestamp(); -// CheckTimestamp(); -// } + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckBinary("localhost:9065", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } -// { -// // drop qnode -// int32_t contLen = sizeof(SMDropQnodeMsg); + { + // drop qnode + int32_t contLen = sizeof(SMDropQnodeMsg); -// SMDropQnodeMsg* pReq = (SMDropQnodeMsg*)rpcMallocCont(contLen); -// pReq->dnodeId = htonl(2); + SMDropQnodeMsg* pReq = (SMDropQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); -// SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, 0); + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); -// test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); -// test.SendShowRetrieveMsg(); -// EXPECT_EQ(test.GetShowRows(), 1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); -// CheckInt16(1); -// CheckBinary("localhost:9071", TSDB_EP_LEN); -// CheckTimestamp(); -// } -// } \ No newline at end of file + CheckInt16(1); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/snode/CMakeLists.txt b/source/dnode/mgmt/impl/test/snode/CMakeLists.txt new file mode 100644 index 0000000000..180b1ec163 --- /dev/null +++ b/source/dnode/mgmt/impl/test/snode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. STEST_SRC) +add_executable(dnode_test_snode ${STEST_SRC}) +target_link_libraries( + dnode_test_snode + PUBLIC sut +) + +add_test( + NAME dnode_test_snode + COMMAND dnode_test_snode +) diff --git a/source/dnode/mgmt/impl/test/snode/snode.cpp b/source/dnode/mgmt/impl/test/snode/snode.cpp new file mode 100644 index 0000000000..a14a575beb --- /dev/null +++ b/source/dnode/mgmt/impl/test/snode/snode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestSnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_snode1", 9066); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9066"; + + server2.Start("/tmp/dnode_test_snode2", fqdn, 9067, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestSnode::test; +TestServer DndTestSnode::server2; + +TEST_F(DndTestSnode, 01_ShowSnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + CHECK_META("show snodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestSnode, 02_Create_Snode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + CHECK_META("show snodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestSnode, 03_Create_Snode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestSnode, 04_Create_Snode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9067); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create snode + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckBinary("localhost:9067", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop snode + int32_t contLen = sizeof(SMDropSnodeMsg); + + SMDropSnodeMsg* pReq = (SMDropSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 6018277a13..2d14e0f92e 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -158,7 +158,7 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pMsg->dnodeId); + pMsg->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -270,7 +270,7 @@ static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBn terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pMsg->dnodeId); + pMsg->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 4afd20e2b6..3b27764c7c 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -158,7 +158,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pMsg->dnodeId); + pMsg->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -270,7 +270,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pMsg->dnodeId); + pMsg->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 083144c3c6..a4825c0e82 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -158,7 +158,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pMsg->dnodeId); + pMsg->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -270,7 +270,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pMsg->dnodeId = htonl(pMsg->dnodeId); + pMsg->dnodeId = htonl(pDnode->id); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); From 3316c1489a25ed75ad847b5b94665dd815a134c6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 05:00:26 -0800 Subject: [PATCH 12/97] test cases --- include/common/tmsg.h | 26 +++--------- source/dnode/mgmt/impl/src/dndMnode.c | 12 +++--- source/dnode/mgmt/impl/test/mnode/mnode.cpp | 16 +++---- source/dnode/mgmt/impl/test/stb/stb.cpp | 4 +- .../dnode/mgmt/impl/test/sut/src/server.cpp | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 32 +++++++------- source/dnode/mnode/impl/src/mndStb.c | 42 +++---------------- source/dnode/mnode/impl/src/mndTopic.c | 6 +-- source/dnode/mnode/impl/src/mndVgroup.c | 3 ++ 9 files changed, 49 insertions(+), 94 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1a88cecf91..95cd82c9fd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -265,19 +265,7 @@ typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; uint64_t suid; - int32_t sverson; - uint32_t ttl; - uint32_t keep; - int32_t numOfTags; - int32_t numOfColumns; - SSchema pSchema[]; -} SCreateStbInternalMsg; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - uint64_t suid; -} SDropStbInternalMsg; +} SVDropStbReq; typedef struct { SMsgHead head; @@ -855,17 +843,13 @@ typedef struct { typedef struct { int32_t dnodeId; -} SCreateMnodeMsg, SDropMnodeMsg; +} SMCreateMnodeMsg, SMDropMnodeMsg, SDDropMnodeMsg; typedef struct { int32_t dnodeId; int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; -} SCreateMnodeInMsg, SAlterMnodeInMsg; - -typedef struct { - int32_t dnodeId; -} SDropMnodeInMsg; +} SDCreateMnodeMsg, SDAlterMnodeMsg; typedef struct { int32_t dnodeId; @@ -1208,13 +1192,13 @@ typedef struct { char* executor; int32_t sqlLen; char* sql; -} SCreateTopicInternalMsg; +} SDCreateTopicMsg; typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; uint64_t tuid; -} SDropTopicInternalMsg; +} SDDropTopicMsg; typedef struct SVCreateTbReq { uint64_t ver; // use a general definition diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 8fb95c0b75..577cb0c3b0 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -299,7 +299,7 @@ static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); } -static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SCreateMnodeInMsg *pMsg) { +static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeMsg *pMsg) { dndInitMnodeOption(pDnode, pOption); pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); @@ -417,8 +417,8 @@ static int32_t dndDropMnode(SDnode *pDnode) { return 0; } -static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { - SCreateMnodeInMsg *pMsg = pRpcMsg->pCont; +static SDCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { + SDCreateMnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); for (int32_t i = 0; i < pMsg->replica; ++i) { pMsg->replicas[i].id = htonl(pMsg->replicas[i].id); @@ -429,7 +429,7 @@ static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { } int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); + SDCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -445,7 +445,7 @@ int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SAlterMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); + SDAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -465,7 +465,7 @@ int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropMnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropMnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mgmt/impl/test/mnode/mnode.cpp index a6cec93539..e9b1ef45bd 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/mnode.cpp @@ -72,9 +72,9 @@ TEST_F(DndTestMnode, 01_ShowDnode) { TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SCreateMnodeMsg); + int32_t contLen = sizeof(SMCreateMnodeMsg); - SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); + SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -85,9 +85,9 @@ TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SCreateMnodeMsg); + int32_t contLen = sizeof(SMCreateMnodeMsg); - SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); + SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -117,9 +117,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { { // create mnode - int32_t contLen = sizeof(SCreateMnodeMsg); + int32_t contLen = sizeof(SMCreateMnodeMsg); - SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); + SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -144,9 +144,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { { // drop mnode - int32_t contLen = sizeof(SDropMnodeMsg); + int32_t contLen = sizeof(SMDropMnodeMsg); - SDropMnodeMsg* pReq = (SDropMnodeMsg*)rpcMallocCont(contLen); + SMDropMnodeMsg* pReq = (SMDropMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_MNODE, pReq, contLen); diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index ca168fb6d7..dca0f48516 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -146,8 +146,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pSchema->bytes = htonl(pSchema->bytes); } - EXPECT_STREQ(pRsp->tbFname, ""); - EXPECT_STREQ(pRsp->stbFname, "1.d1.stb"); + EXPECT_STREQ(pRsp->tbFname, "1.d1.stb"); + EXPECT_STREQ(pRsp->stbFname, ""); EXPECT_EQ(pRsp->numOfColumns, 2); EXPECT_EQ(pRsp->numOfTags, 3); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index f29b2fad1d..8ac5f62144 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -26,7 +26,7 @@ SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t p SDnodeOpt option = {0}; option.sver = 1; option.numOfCores = 1; - option.numOfSupportVnodes = 1; + option.numOfSupportVnodes = 16; option.numOfCommitThreads = 1; option.statusInterval = 1; option.numOfThreadsPerCore = 1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 04e77c0136..b76d72d79c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -251,7 +251,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno void *pIter = NULL; int32_t numOfReplicas = 0; - SCreateMnodeInMsg createMsg = {0}; + SDCreateMnodeMsg createMsg = {0}; while (1) { SMnodeObj *pMObj = NULL; pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); @@ -281,18 +281,18 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno STransAction action = {0}; - SAlterMnodeInMsg *pMsg = malloc(sizeof(SAlterMnodeInMsg)); + SDAlterMnodeMsg *pMsg = malloc(sizeof(SDAlterMnodeMsg)); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; } - memcpy(pMsg, &createMsg, sizeof(SAlterMnodeInMsg)); + memcpy(pMsg, &createMsg, sizeof(SDAlterMnodeMsg)); pMsg->dnodeId = htonl(pMObj->id); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pMsg; - action.contLen = sizeof(SAlterMnodeInMsg); + action.contLen = sizeof(SDAlterMnodeMsg); action.msgType = TDMT_DND_ALTER_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -309,14 +309,14 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - SCreateMnodeInMsg *pMsg = malloc(sizeof(SCreateMnodeInMsg)); + SDCreateMnodeMsg *pMsg = malloc(sizeof(SDCreateMnodeMsg)); if (pMsg == NULL) return -1; - memcpy(pMsg, &createMsg, sizeof(SAlterMnodeInMsg)); + memcpy(pMsg, &createMsg, sizeof(SDAlterMnodeMsg)); pMsg->dnodeId = htonl(pObj->id); action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pMsg; - action.contLen = sizeof(SCreateMnodeInMsg); + action.contLen = sizeof(SDCreateMnodeMsg); action.msgType = TDMT_DND_CREATE_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -327,7 +327,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } -static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SCreateMnodeMsg *pCreate) { +static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateMnodeMsg *pCreate) { SMnodeObj mnodeObj = {0}; mnodeObj.id = pDnode->id; mnodeObj.createdTime = taosGetTimestampMs(); @@ -370,7 +370,7 @@ CREATE_MNODE_OVER: static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont; + SMCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont; pCreate->dnodeId = htonl(pCreate->dnodeId); @@ -423,7 +423,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode void *pIter = NULL; int32_t numOfReplicas = 0; - SAlterMnodeInMsg alterMsg = {0}; + SDAlterMnodeMsg alterMsg = {0}; while (1) { SMnodeObj *pMObj = NULL; pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); @@ -449,18 +449,18 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (pMObj->id != pObj->id) { STransAction action = {0}; - SAlterMnodeInMsg *pMsg = malloc(sizeof(SAlterMnodeInMsg)); + SDAlterMnodeMsg *pMsg = malloc(sizeof(SDAlterMnodeMsg)); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; } - memcpy(pMsg, &alterMsg, sizeof(SAlterMnodeInMsg)); + memcpy(pMsg, &alterMsg, sizeof(SDAlterMnodeMsg)); pMsg->dnodeId = htonl(pMObj->id); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pMsg; - action.contLen = sizeof(SAlterMnodeInMsg); + action.contLen = sizeof(SDAlterMnodeMsg); action.msgType = TDMT_DND_ALTER_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -478,7 +478,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - SDropMnodeInMsg *pMsg = malloc(sizeof(SDropMnodeInMsg)); + SDDropMnodeMsg *pMsg = malloc(sizeof(SDDropMnodeMsg)); if (pMsg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -487,7 +487,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pMsg; - action.contLen = sizeof(SDropMnodeInMsg); + action.contLen = sizeof(SDDropMnodeMsg); action.msgType = TDMT_DND_DROP_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -537,7 +537,7 @@ DROP_MNODE_OVER: static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SMDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("mnode:%d, start to drop", pDrop->dnodeId); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index af18c814ca..918f43f2bd 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -201,7 +201,6 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { } static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int *pContLen) { -#if 1 SVCreateTbReq req; void * buf; int bsize; @@ -235,43 +234,12 @@ static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb *pContLen = sizeof(SMsgHead) + bsize; return buf; - -#else - int32_t totalCols = pStb->numOfTags + pStb->numOfColumns; - int32_t contLen = totalCols * sizeof(SSchema) + sizeof(SCreateStbInternalMsg); - - SCreateStbInternalMsg *pCreate = calloc(1, contLen); - if (pCreate == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pCreate->head.contLen = htonl(contLen); - pCreate->head.vgId = htonl(pVgroup->vgId); - memcpy(pCreate->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pCreate->suid = htobe64(pStb->uid); - pCreate->sverson = htonl(pStb->version); - pCreate->ttl = 0; - pCreate->keep = 0; - pCreate->numOfTags = htonl(pStb->numOfTags); - pCreate->numOfColumns = htonl(pStb->numOfColumns); - - memcpy(pCreate->pSchema, pStb->pSchema, totalCols * sizeof(SSchema)); - for (int32_t t = 0; t < totalCols; ++t) { - SSchema *pSchema = &pCreate->pSchema[t]; - pSchema->bytes = htonl(pSchema->bytes); - pSchema->colId = htonl(pSchema->colId); - } - - *pContLen = contLen; - return pCreate; -#endif } -static SDropStbInternalMsg *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SDropStbInternalMsg); +static SVDropStbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t contLen = sizeof(SVDropStbReq); - SDropStbInternalMsg *pDrop = calloc(1, contLen); + SVDropStbReq *pDrop = calloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -402,7 +370,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SDropStbInternalMsg *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); + SVDropStbReq *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -413,7 +381,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pMsg; - action.contLen = sizeof(SDropStbInternalMsg); + action.contLen = sizeof(SVDropStbReq); action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 49c96967e6..24e32e07b4 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -162,10 +162,10 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { return mndAcquireDb(pMnode, db); } -static SDropTopicInternalMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { - int32_t contLen = sizeof(SDropTopicInternalMsg); +static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { + int32_t contLen = sizeof(SDDropTopicMsg); - SDropTopicInternalMsg *pDrop = calloc(1, contLen); + SDDropTopicMsg *pDrop = calloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 78205d5f6a..23b62d2dfb 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -120,6 +120,9 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SDB_GET_INT32(pRaw, pRow, dataPos, &pVgid->dnodeId) + if (pVgroup->replica == 1) { + pVgid->role = TAOS_SYNC_STATE_LEADER; + } } SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_VGROUP_RESERVE_SIZE) From 899bd183a642665807d513ae8d9036376bf170c5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 05:03:08 -0800 Subject: [PATCH 13/97] test cases --- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index ec2c2d9a44..96bb6e5390 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -68,7 +68,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); - CheckInt16(1); + CheckInt16(16); CheckBinary("ready", 10); CheckTimestamp(); CheckBinary("", 24); @@ -112,8 +112,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("localhost:9042", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); - CheckInt16(1); - CheckInt16(1); + CheckInt16(16); + CheckInt16(16); CheckBinary("ready", 10); CheckBinary("ready", 10); CheckTimestamp(); @@ -140,7 +140,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); - CheckInt16(1); + CheckInt16(16); CheckBinary("ready", 10); CheckTimestamp(); CheckBinary("", 24); @@ -199,10 +199,10 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(0); CheckInt16(0); CheckInt16(0); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); CheckBinary("ready", 10); CheckBinary("ready", 10); CheckBinary("ready", 10); @@ -242,10 +242,10 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(0); CheckInt16(0); CheckInt16(0); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); CheckBinary("ready", 10); CheckBinary("ready", 10); CheckBinary("ready", 10); From 261ef100f7956df40103b9f95d603ea6e926ac8a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Dec 2021 22:52:15 +0800 Subject: [PATCH 14/97] refactor code --- source/libs/index/src/index_cache.c | 49 ++++++++++++++----- .../index/src/index_fst_counting_writer.c | 7 ++- source/libs/index/src/index_tfile.c | 6 +-- source/libs/index/test/indexTests.cc | 29 +++++------ 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 9807969fd1..882091afb9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 100 +#define MEM_TERM_LIMIT 200 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -66,19 +66,43 @@ void indexCacheDebug(IndexCache* cache) { indexMemRef(tbl); pthread_mutex_unlock(&cache->mtx); - SSkipList* slt = tbl->mem; - SSkipListIterator* iter = tSkipListCreateIter(slt); - while (tSkipListIterNext(iter)) { - SSkipListNode* node = tSkipListIterGet(iter); - CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - if (ct != NULL) { - // TODO, add more debug info - indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + { + SSkipList* slt = tbl->mem; + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } } - } - tSkipListDestroyIter(iter); + tSkipListDestroyIter(iter); - indexMemUnRef(tbl); + indexMemUnRef(tbl); + } + + { + pthread_mutex_lock(&cache->mtx); + tbl = cache->imm; + indexMemRef(tbl); + pthread_mutex_unlock(&cache->mtx); + if (tbl != NULL) { + SSkipList* slt = tbl->mem; + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } + } + tSkipListDestroyIter(iter); + } + + indexMemUnRef(tbl); + } } void indexCacheDestroySkiplist(SSkipList* slt) { @@ -247,6 +271,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + indexCacheDebug(pCache); int ret = indexQueryMem(mem, &ct, qtype, result, s); if (ret == 0 && *s != kTypeDeletion) { diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index fe2a5c767a..710db563d9 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -52,7 +52,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // tfFsync(ctx->fd); + tfFsync(ctx->file.fd); // tfFlush(ctx->file.fd); } else { // do nothing @@ -101,7 +101,10 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { tfClose(ctx->file.fd); - if (remove) unlink(ctx->file.buf); + if (remove) { + indexError("rm file %s", ctx->file.buf); + unlink(ctx->file.buf); + } } free(ctx); } diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 2d7b571a37..17322e301e 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -81,7 +81,7 @@ TFileCache* tfileCacheCreate(const char* path) { continue; } - WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 1024 * 64); if (wc == NULL) { indexError("failed to open index:%s", file); goto End; @@ -211,7 +211,7 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c char fullname[256] = {0}; snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); - WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024); + WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); if (wcx == NULL) { return NULL; } TFileHeader tfh = {0}; @@ -229,7 +229,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c char fullname[256] = {0}; snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); - WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024); + WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); if (wc == NULL) { return NULL; } TFileReader* reader = tfileReaderCreateImpl(wc); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 3d06dde98f..588205c67f 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -427,7 +427,7 @@ static TFileValue* genTFileValue(const char* val) { memcpy(tv->colVal, val, vlen); tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); - for (size_t i = 0; i < 10; i++) { + for (size_t i = 0; i < 200; i++) { uint64_t v = i; taosArrayPush(tv->tableId, &v); } @@ -440,17 +440,14 @@ static void destroyTFileValue(void* val) { free(tv); } TEST_F(IndexTFileEnv, test_tfile_write) { - TFileValue* v1 = genTFileValue("c"); - TFileValue* v2 = genTFileValue("ab"); - TFileValue* v3 = genTFileValue("b"); - TFileValue* v4 = genTFileValue("d"); + TFileValue* v1 = genTFileValue("ab"); SArray* data = (SArray*)taosArrayInit(4, sizeof(void*)); taosArrayPush(data, &v1); - taosArrayPush(data, &v2); - taosArrayPush(data, &v3); - taosArrayPush(data, &v4); + // taosArrayPush(data, &v2); + // taosArrayPush(data, &v3); + // taosArrayPush(data, &v4); fObj->Put(data); for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); } @@ -464,7 +461,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); - assert(taosArrayGetSize(result) == 10); + assert(taosArrayGetSize(result) == 200); indexTermDestroy(term); // tfileWriterDestroy(twrite); @@ -665,8 +662,8 @@ class IndexObj { } ~IndexObj() { - indexClose(idx); indexCleanUp(); + indexClose(idx); } private: @@ -696,9 +693,9 @@ TEST_F(IndexEnv2, testIndexOpen) { exit(1); } - int targetSize = 100; + int targetSize = 200; { - std::string colName("tag1"), colVal("Hello world"); + std::string colName("tag1"), colVal("Hello"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -712,8 +709,8 @@ TEST_F(IndexEnv2, testIndexOpen) { indexMultiTermDestroy(terms); } { - size_t size = 100; - std::string colName("tag1"), colVal("hello world"); + size_t size = 200; + std::string colName("tag1"), colVal("hello"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -728,7 +725,7 @@ TEST_F(IndexEnv2, testIndexOpen) { } { - std::string colName("tag1"), colVal("Hello world"); + std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), @@ -747,7 +744,7 @@ TEST_F(IndexEnv2, testIndex_TrigeFlush) { if (index->Init(path) != 0) {} int numOfTable = 100 * 10000; index->WriteMillonData("tag1", "Hello world", numOfTable); - int target = index->SearchOne("tag1", "Hellow world"); + int target = index->SearchOne("tag1", "Hello world"); assert(numOfTable == target); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { From be7bccae63fd57dfacef8e1bd675845473084e5f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Dec 2021 23:07:01 +0800 Subject: [PATCH 15/97] [td-11818] support show tables; --- include/common/tmsg.h | 4 +- include/common/tname.h | 3 + include/libs/parser/parser.h | 1 - include/libs/qworker/qworker.h | 5 +- source/client/src/clientImpl.c | 8 +- source/client/src/clientMsgHandler.c | 106 ++++++++++++++++------ source/client/test/clientTests.cpp | 68 ++++++++------ source/common/src/tname.c | 10 ++ source/dnode/mgmt/impl/src/dndTransport.c | 2 + source/dnode/vnode/impl/src/vnodeQuery.c | 6 +- source/libs/parser/src/astToMsg.c | 1 - source/libs/parser/src/dCDAstProcess.c | 103 ++++++++++++--------- source/libs/parser/src/parserUtil.c | 10 -- source/libs/qworker/src/qworker.c | 89 ++++++++++++++++-- 14 files changed, 291 insertions(+), 125 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b386b729d4..0ed2e51649 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1245,13 +1245,13 @@ typedef struct SVShowTablesReq { } SVShowTablesReq; typedef struct SVShowTablesRsp { - int64_t id; + int32_t id; STableMetaMsg metaInfo; } SVShowTablesRsp; typedef struct SVShowTablesFetchReq { SMsgHead head; - int64_t id; + int32_t id; } SVShowTablesFetchReq; typedef struct SVShowTablesFetchRsp { diff --git a/include/common/tname.h b/include/common/tname.h index 3ac7f8b27b..11d97dac06 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -17,6 +17,7 @@ #define TDENGINE_TNAME_H #include "tdef.h" +#include "tmsg.h" #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 @@ -58,4 +59,6 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, int32_t acctId); +SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); + #endif // TDENGINE_TNAME_H diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index dd3d92866f..a9e1f26d20 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -74,7 +74,6 @@ int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo); STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); -SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); int32_t getNewResColId(); void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 83047a44de..9897467230 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -54,8 +54,11 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); -void qWorkerDestroy(void **qWorkerMgmt); +int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); +int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + +void qWorkerDestroy(void **qWorkerMgmt); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6ecd9a59f7..4435deb9b3 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -480,7 +480,13 @@ void* doFetchRow(SRequestObj* pRequest) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { - pRequest->type = TDMT_MND_SHOW_RETRIEVE; + if (pRequest->type == TDMT_MND_SHOW) { + pRequest->type = TDMT_MND_SHOW_RETRIEVE; + } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { + pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + } else { + // do nothing + } SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e2fdf96385..76f1c8370d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -74,39 +74,41 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -static int32_t buildRetrieveMnodeMsg(SRequestObj *pRequest, SMsgSendInfo* pMsgSendInfo) { - pMsgSendInfo->msgType = TDMT_MND_SHOW_RETRIEVE; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); - pMsgSendInfo->requestObjRefId = pRequest->self; - pMsgSendInfo->param = pRequest; - pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; - - SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); - if (pRetrieveMsg == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pRetrieveMsg->showId = htonl(pRequest->body.execId); - pMsgSendInfo->msgInfo.pData = pRetrieveMsg; - return TSDB_CODE_SUCCESS; -} - SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { - buildRetrieveMnodeMsg(pRequest, pMsgSendInfo); + pMsgSendInfo->requestObjRefId = pRequest->self; + pMsgSendInfo->requestId = pRequest->requestId; + pMsgSendInfo->param = pRequest; + pMsgSendInfo->msgType = pRequest->type; + + if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { + if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { + SRetrieveTableMsg* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); + if (pRetrieveMsg == NULL) { + return NULL; + } + + pRetrieveMsg->showId = htonl(pRequest->body.execId); + pMsgSendInfo->msgInfo.pData = pRetrieveMsg; + pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); + } else { + SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); + if (pFetchMsg == NULL) { + return NULL; + } + + pFetchMsg->id = htonl(pRequest->body.execId); + pFetchMsg->head.vgId = htonl(13); + pMsgSendInfo->msgInfo.pData = pFetchMsg; + pMsgSendInfo->msgInfo.len = sizeof(SVShowTablesFetchReq); + } } else { assert(pRequest != NULL); - pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->msgInfo = pRequest->body.requestMsg; - pMsgSendInfo->msgType = pRequest->type; - pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->param = pRequest; - - pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; } + pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; return pMsgSendInfo; } @@ -128,6 +130,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pMetaMsg->tuid = htobe64(pMetaMsg->tuid); for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { pSchema->bytes = htonl(pSchema->bytes); + pSchema->colId = htonl(pSchema->colId); pSchema++; } @@ -157,8 +160,16 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) assert(pMsg->len >= sizeof(SRetrieveTableRsp)); SRequestObj* pRequest = param; -// tfree(pRequest->body.resInfo.pRspMsg); -// pRequest->body.resInfo.pRspMsg = pMsg->pData; + tfree(pRequest->body.resInfo.pRspMsg); + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + terrno = code; + tsem_post(&pRequest->body.rspSem); + return code; + } + + pRequest->body.resInfo.pRspMsg = pMsg->pData; SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg->pData; pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); @@ -169,7 +180,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) tfree(pResInfo->pRspMsg); pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; - pResInfo->pData = pRetrieve->data; // todo fix this in async model + pResInfo->pData = pRetrieve->data; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); @@ -181,6 +192,42 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) return 0; } +int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { + assert(pMsg->len >= sizeof(SRetrieveTableRsp)); + + SRequestObj* pRequest = param; + tfree(pRequest->body.resInfo.pRspMsg); + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + terrno = code; + tsem_post(&pRequest->body.rspSem); + return code; + } + + pRequest->body.resInfo.pRspMsg = pMsg->pData; + + SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *) pMsg->pData; + pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows); + pFetchRsp->precision = htons(pFetchRsp->precision); + + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + + tfree(pResInfo->pRspMsg); + pResInfo->pRspMsg = pMsg->pData; + pResInfo->numOfRows = pFetchRsp->numOfRows; + pResInfo->pData = pFetchRsp->data; + + pResInfo->current = 0; + setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); + + tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pFetchRsp->numOfRows, + pFetchRsp->completed, pRequest->body.execId); + + tsem_post(&pRequest->body.rspSem); + return 0; +} + int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { // todo rsp with the vnode id list SRequestObj* pRequest = param; @@ -297,4 +344,7 @@ void initMsgHandleFp() { handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; + + handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = processShowRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = processRetrieveVndRsp; } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index be001780ca..72867b283a 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -226,25 +226,25 @@ TEST(testCase, use_db_test) { } TEST(testCase, drop_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - showDB(pConn); - - TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - showDB(pConn); - - pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("create to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - taos_close(pConn); +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// showDB(pConn); +// +// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// showDB(pConn); +// +// pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// taos_close(pConn); } TEST(testCase, create_stable_Test) { @@ -301,12 +301,12 @@ TEST(testCase, create_ctable_Test) { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); taos_close(pConn); } @@ -441,7 +441,23 @@ TEST(testCase, show_table_Test) { taos_free_result(pRes); pRes = taos_query(pConn, "show tables"); - taos_free_result(pRes); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); taos_close(pConn); } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 88a5ebb7f6..f8ef9f0979 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -259,3 +259,13 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return 0; } + +SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { + SSchema s = {0}; + s.type = type; + s.bytes = bytes; + s.colId = colId; + + tstrncpy(s.name, name, tListLen(s.name)); + return s; +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8db09ea4cd..1804a11c76 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -123,6 +123,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; } static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 1c6924040c..6632676d8b 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -36,12 +36,14 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_DROP_TASK: return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_SHOW_TABLES: + return qWorkerProcessShowMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_SHOW_TABLES_FETCH: + return qWorkerProcessShowFetchMsg(pVnode, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; - break; } - return 0; } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index bdbc095861..6b0e0ed828 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -90,7 +90,6 @@ SShowMsg* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); pShowMsg->type = pShowInfo->showType; - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { SToken* pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 7160b13eba..8a09ea9ed8 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { } static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, - SMsgBuf* pMsgBuf) { + SEpSet* pEpSet, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid name"; const char* msg2 = "wildcard string should be less than %d characters"; const char* msg3 = "database name too long"; @@ -31,57 +31,69 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou * wildcard in like clause in pInfo->pMiscInfo->a[1] */ int16_t showType = pShowInfo->showType; - if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { - SToken* pDbPrefixToken = &pShowInfo->prefix; - if (pDbPrefixToken->type != 0) { - if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long - return buildInvalidOperationMsg(pMsgBuf, msg3); + if (showType == TSDB_MGMT_TABLE_TABLE) { + SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + *pEpSet = pCtx->mgmtEpSet; + + // catalogGetDBVgroupVersion() + pShowReq->head.vgId = htonl(13); + *outputLen = sizeof(SVShowTablesReq); + *output = pShowReq; + } else { + if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { + SToken* pDbPrefixToken = &pShowInfo->prefix; + if (pDbPrefixToken->type != 0) { + if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (pDbPrefixToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); + // if (ret != TSDB_CODE_SUCCESS) { + // return buildInvalidOperationMsg(pMsgBuf, msg1); + // } } - if (pDbPrefixToken->n <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg5); + // show table/stable like 'xxxx', set the like pattern for show tables + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type != 0) { + if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + pPattern->n = strdequote(pPattern->z); + if (pPattern->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + if (pPattern->n > tsMaxWildCardsLen) { + char tmp[64] = {0}; + sprintf(tmp, msg2, tsMaxWildCardsLen); + return buildInvalidOperationMsg(pMsgBuf, tmp); + } + } + } else if (showType == TSDB_MGMT_TABLE_VNODES) { + if (pShowInfo->prefix.type == 0) { + return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); } - if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } - - // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); - // if (ret != TSDB_CODE_SUCCESS) { - // return buildInvalidOperationMsg(pMsgBuf, msg1); - // } - } - - // show table/stable like 'xxxx', set the like pattern for show tables - SToken* pPattern = &pShowInfo->pattern; - if (pPattern->type != 0) { - if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { - return buildInvalidOperationMsg(pMsgBuf, msg4); - } - - pPattern->n = strdequote(pPattern->z); - if (pPattern->n <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg6); - } - - if (pPattern->n > tsMaxWildCardsLen) { - char tmp[64] = {0}; - sprintf(tmp, msg2, tsMaxWildCardsLen); - return buildInvalidOperationMsg(pMsgBuf, tmp); + if (pShowInfo->prefix.type == TK_STRING) { + pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); } } - } else if (showType == TSDB_MGMT_TABLE_VNODES) { - if (pShowInfo->prefix.type == 0) { - return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); - } - if (pShowInfo->prefix.type == TK_STRING) { - pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); - } + *pEpSet = pCtx->mgmtEpSet; + *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len); + *outputLen = sizeof(SShowMsg) /* + htons(pShowMsg->payloadLen)*/; } - *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len); - *outputLen = sizeof(SShowMsg) /* + htons(pShowMsg->payloadLen)*/; return TSDB_CODE_SUCCESS; } @@ -608,8 +620,9 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } case TSDB_SQL_SHOW: { - code = setShowInfo(&pInfo->pMiscInfo->showOpt, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, pMsgBuf); - pDcl->msgType = TDMT_MND_SHOW; + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; + code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, pMsgBuf); + pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE)? TDMT_VND_SHOW_TABLES:TDMT_MND_SHOW; break; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 20f330247e..a454061fd7 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -563,16 +563,6 @@ TAOS_FIELD createField(const SSchema* pSchema) { return f; } -SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name) { - SSchema s = {0}; - s.type = type; - s.bytes = bytes; - s.colId = colId; - - tstrncpy(s.name, name, tListLen(s.name)); - return s; -} - void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema) { pColumn->uid = uid; pColumn->flag = flag; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 4296e82a56..d5ecd40ccd 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,8 +1,9 @@ -#include "tmsg.h" -#include "query.h" #include "qworker.h" -#include "qworkerInt.h" +#include "tname.h" #include "planner.h" +#include "query.h" +#include "qworkerInt.h" +#include "tmsg.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -634,7 +635,6 @@ int32_t qwBuildAndSendFetchRsp(SRpcMsg *pMsg, void *data) { return TSDB_CODE_SUCCESS; } - int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp)); pRsp->code = code; @@ -665,11 +665,68 @@ int32_t qwBuildAndSendDropRsp(SRpcMsg *pMsg, int32_t code) { }; rpcSendResponse(&rpcRsp); - return TSDB_CODE_SUCCESS; } +int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { + int32_t numOfCols = 6; + SVShowTablesRsp *pRsp = (SVShowTablesRsp *)rpcMallocCont(sizeof(SVShowTablesRsp) + sizeof(SSchema) * numOfCols); + int32_t cols = 0; + SSchema *pSchema = pRsp->metaInfo.pSchema; + + const SSchema *s = tGetTbnameColumnSchema(); + *pSchema = createSchema(s->type, htonl(s->bytes), htonl(cols++), "name"); + pSchema++; + + int32_t type = TSDB_DATA_TYPE_TIMESTAMP; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "created"); + pSchema++; + + type = TSDB_DATA_TYPE_SMALLINT; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "columns"); + pSchema++; + + *pSchema = createSchema(s->type, htonl(s->bytes), htonl(cols++), "stable"); + pSchema++; + + type = TSDB_DATA_TYPE_BIGINT; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "uid"); + pSchema++; + + type = TSDB_DATA_TYPE_INT; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "vgId"); + + pRsp->metaInfo.numOfColumns = htonl(cols); + + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pRsp, + .contLen = sizeof(*pRsp), + .code = code, + }; + + rpcSendResponse(&rpcMsg); + return TSDB_CODE_SUCCESS; +} + +int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) { + SVShowTablesFetchRsp *pRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp)); + int32_t handle = htonl(pFetchReq->id); + + pRsp->numOfRows = 0; + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pRsp, + .contLen = sizeof(*pRsp), + .code = 0, + }; + + rpcSendResponse(&rpcMsg); + return TSDB_CODE_SUCCESS; +} int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) { SQWSchStatus *sch = NULL; @@ -801,7 +858,6 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI return TSDB_CODE_SUCCESS; } - int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -911,7 +967,6 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } - int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { @@ -1157,6 +1212,25 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + int32_t code = 0; + SVShowTablesReq *pReq = pMsg->pCont; + QW_ERR_RET(qwBuildAndSendShowRsp(pMsg, code)); +} + +int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SVShowTablesFetchReq *pFetchReq = pMsg->pCont; + QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); +} + int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; int8_t status = 0; @@ -1182,7 +1256,6 @@ int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_RET(code); } - void qWorkerDestroy(void **qWorkerMgmt) { if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { return; From dd4af3274d8e8e4267bd5554b0c8755a8989f5e8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 17:56:39 -0800 Subject: [PATCH 16/97] fix crash while run CI --- source/dnode/mgmt/impl/src/dndTransport.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index ba93d1913d..b726743c9a 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -383,6 +383,7 @@ void dndCleanupTrans(SDnode *pDnode) { void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pMsg) { STransMgmt *pMgmt = &pDnode->tmgmt; + if (pMgmt->clientRpc == NULL) return; rpcSendRequest(pMgmt->clientRpc, pEpSet, pMsg, NULL); } From 01df1cbe5b1dbe3a90c7534733c3cdb19895844e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 10:26:41 +0800 Subject: [PATCH 17/97] [td-11818]fix bug in show --- source/client/src/clientMsgHandler.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 76f1c8370d..f30a0d10d7 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -177,7 +177,6 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) SReqResultInfo* pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; From 07a3680812bfecc1498e56b91a692663adb5b779 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 10:28:01 +0800 Subject: [PATCH 18/97] [td-11818] refactor. --- source/client/src/clientMsgHandler.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f30a0d10d7..f4cec5c627 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -160,8 +160,9 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) assert(pMsg->len >= sizeof(SRetrieveTableRsp)); SRequestObj* pRequest = param; - tfree(pRequest->body.resInfo.pRspMsg); + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + tfree(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; terrno = code; @@ -169,14 +170,10 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) return code; } - pRequest->body.resInfo.pRspMsg = pMsg->pData; - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg->pData; pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); pRetrieve->precision = htons(pRetrieve->precision); - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; From 7ad701cc5e48681226cccecf4760f5a306c836ca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 10:37:39 +0800 Subject: [PATCH 19/97] [td-11818] add rsp error check. --- source/client/src/clientMsgHandler.c | 53 ++++++++++++++++++---------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f4cec5c627..9ee890afd9 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -22,19 +22,23 @@ int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); +static void setErrno(SRequestObj* pRequest, int32_t code) { + pRequest->code = code; + terrno = code; +} + int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; - pRequest->code = code; + setErrno(pRequest, code); + sem_post(&pRequest->body.rspSem); - return 0; + return code; } int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - terrno = code; - + setErrno(pRequest, code); sem_post(&pRequest->body.rspSem); return code; } @@ -115,7 +119,7 @@ SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; } @@ -157,19 +161,18 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { - assert(pMsg->len >= sizeof(SRetrieveTableRsp)); - - SRequestObj* pRequest = param; - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - + SRequestObj *pRequest = param; + SReqResultInfo *pResInfo = &pRequest->body.resInfo; tfree(pResInfo->pRspMsg); + if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - terrno = code; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; } + assert(pMsg->len >= sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg->pData; pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); pRetrieve->precision = htons(pRetrieve->precision); @@ -195,8 +198,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { tfree(pRequest->body.resInfo.pRspMsg); if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - terrno = code; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; } @@ -209,7 +211,6 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { SReqResultInfo* pResInfo = &pRequest->body.resInfo; - tfree(pResInfo->pRspMsg); pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pFetchRsp->numOfRows; pResInfo->pData = pFetchRsp->data; @@ -231,6 +232,14 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { + SRequestObj* pRequest = param; + + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg->pData; SName name = {0}; tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); @@ -238,17 +247,23 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { char db[TSDB_DB_NAME_LEN] = {0}; tNameGetDbName(&name, db); - SRequestObj* pRequest = param; setConnectionDB(pRequest->pTscObj, db); - tsem_post(&pRequest->body.rspSem); return 0; } int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { - assert(pMsg != NULL); + assert(pMsg != NULL && param != NULL); SRequestObj* pRequest = param; + + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + tsem_post(&pRequest->body.rspSem); + return code; } int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { From 6a55262ba6255845e9a99e5ac37e6667736a2726 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 10:38:32 +0800 Subject: [PATCH 20/97] [td-11818]check error in rsp. --- source/client/src/clientMsgHandler.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9ee890afd9..07a7286173 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -269,7 +269,14 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { // todo: Remove cache in catalog cache. SRequestObj* pRequest = param; + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + tsem_post(&pRequest->body.rspSem); + return code; } void initMsgHandleFp() { From a1f65e9c4f7f25c82606c4a98a4d297080a573a1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 10:39:41 +0800 Subject: [PATCH 21/97] [td-11818] refactor. --- source/client/src/clientMsgHandler.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 07a7286173..6575102f81 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -18,7 +18,6 @@ #include "tname.h" #include "clientInt.h" #include "clientLog.h" -#include "trpc.h" int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); From 8ca61bdf48193e8ffb02d7ffb8e6c638e8399864 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 10:43:29 +0800 Subject: [PATCH 22/97] [td-11818] fix compiler warning. --- source/util/test/hashTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index d31fcfb7ef..ac1bae2434 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -154,9 +154,9 @@ void acquireRleaseTest() { int32_t code = 0; int32_t num = 0; TESTSTRUCT data = {0}; - char *str1 = "abcdefg"; - char *str2 = "aaaaaaa"; - char *str3 = "123456789"; + const char *str1 = "abcdefg"; + const char *str2 = "aaaaaaa"; + const char *str3 = "123456789"; data.p = (char *)malloc(10); strcpy(data.p, str1); From da058b47c4e707e957ffd7267f4baae3ea9680f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 02:46:50 +0000 Subject: [PATCH 23/97] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 51e3330ebf..4ef7dc01a4 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -444,6 +444,7 @@ int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { void * pBuf; int tlen; STableMetaMsg *pMsg; + SSchema * pSchema; key.data = tbname; key.size = strlen(tbname) + 1; @@ -487,10 +488,9 @@ int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { pMsg->tversion = 0; pMsg->suid = tbCfg.stbCfg.suid; pMsg->tuid = tbCfg.stbCfg.suid; - for (size_t i = 0; i < tbCfg.stbCfg.nTagCols; i++) { - - } - + memcpy(pMsg->pSchema, tbCfg.stbCfg.pSchema, sizeof(SSchema) * tbCfg.stbCfg.nCols); + memcpy(POINTER_SHIFT(pMsg->pSchema, sizeof(SSchema) * tbCfg.stbCfg.nCols), tbCfg.stbCfg.pTagSchema, + sizeof(SSchema) * tbCfg.stbCfg.nTagCols); break; case META_CHILD_TABLE: ASSERT(0); From 24c6abcb0b2f34927ea10e9394edff7e81d5fc4d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 03:32:05 +0000 Subject: [PATCH 24/97] add another extention --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8132eab539..fd0ada95d9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,7 +18,8 @@ "ms-vscode.cpptools", "ms-vscode.cmake-tools", "austin.code-gnu-global", - "visualstudioexptteam.vscodeintel" + "visualstudioexptteam.vscodeintel", + "eamodio.gitlens" ], // Use 'forwardPorts' to make a list of ports inside the container available locally. From b2cf6b34965937d6f0c258afc37562113e67970a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 19:35:46 -0800 Subject: [PATCH 25/97] add support vnodes config --- include/common/tglobal.h | 69 ++++---- include/common/tmsg.h | 4 +- include/dnode/mgmt/dnode.h | 4 +- include/os/osEnv.h | 1 - source/common/src/tglobal.c | 172 ++++++++----------- source/dnode/mgmt/daemon/src/daemon.c | 2 +- source/dnode/mgmt/impl/src/dndDnode.c | 4 +- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 2 +- source/dnode/mgmt/impl/test/sut/src/base.cpp | 1 - source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/src/mndDnode.c | 6 +- source/dnode/mnode/impl/src/mndVgroup.c | 2 +- source/util/src/tlog.c | 1 - tests/script/sh/deploy.sh | 1 + tests/script/tmp/dnodes.sim | 124 +++++++------ 15 files changed, 192 insertions(+), 205 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 42956b6bdd..e8a56b77c9 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -30,11 +30,12 @@ extern char tsLocalEp[]; extern uint16_t tsServerPort; extern int32_t tsStatusInterval; extern int8_t tsEnableTelemetryReporting; +extern int32_t tsNumOfSupportVnodes; // common extern int tsRpcTimer; extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled +extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; @@ -48,14 +49,18 @@ extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; extern char tsTempDir[]; -extern int64_t tsMaxVnodeQueuedBytes; -extern int tsCompatibleModel; // 2.0 compatible model +extern int tsCompatibleModel; // 2.0 compatible model +extern int8_t tsEnableSlaveQuery; +extern int8_t tsEnableAdjustMaster; +extern int8_t tsPrintAuth; +extern int64_t tsTickPerDay[3]; -//query buffer management -extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing -extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing -extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked -extern int8_t tsKeepOriginalColumnName; +// query buffer management +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing +extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node +extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked +extern int8_t tsKeepOriginalColumnName; +extern int8_t tsDeadLockKillQuery; // client extern int32_t tsMaxSQLStringLen; @@ -72,27 +77,17 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// balance -extern int8_t tsEnableSlaveQuery; - - -// interna -extern int8_t tsPrintAuth; -extern char tsVnodeDir[]; -extern char tsMnodeDir[]; -extern int64_t tsTickPerDay[3]; - // system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; +extern float tsTotalLogDirGB; +extern float tsTotalTmpDirGB; +extern float tsTotalDataDirGB; +extern float tsAvailLogDirGB; +extern float tsAvailTmpDirectorySpace; +extern float tsAvailDataDirGB; +extern float tsUsedDataDirGB; +extern float tsMinimalLogDirGB; +extern float tsReservedTmpDirectorySpace; +extern float tsMinimalDataDirGB; extern uint32_t tsVersion; // build info @@ -102,17 +97,13 @@ extern char gitinfo[]; extern char gitinfoOfInternal[]; extern char buildinfo[]; -#ifdef TD_TSZ -// lossy -extern char lossyColumns[]; -extern double fPrecision; -extern double dPrecision; -extern uint32_t maxRange; -extern uint32_t curRange; -extern char Compressor[]; -#endif -// long query -extern int8_t tsDeadLockKillQuery; +// lossy +extern char tsLossyColumns[]; +extern double tsFPrecision; +extern double tsDPrecision; +extern uint32_t tsMaxRange; +extern uint32_t tsCurRange; +extern char tsCompressor[]; typedef struct { char dir[TSDB_FILENAME_LEN]; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 95cd82c9fd..c19e8d0f92 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -651,8 +651,8 @@ typedef struct { int64_t clusterId; int64_t rebootTime; int64_t updateTime; - int16_t numOfCores; - int16_t numOfSupportVnodes; + int32_t numOfCores; + int32_t numOfSupportVnodes; char dnodeEp[TSDB_EP_LEN]; SClusterCfg clusterCfg; SVnodeLoads vnodeLoads; diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e12ce09422..2e3863c3a1 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -27,8 +27,8 @@ typedef struct SDnode SDnode; typedef struct { int32_t sver; - int16_t numOfCores; - int16_t numOfSupportVnodes; + int32_t numOfCores; + int32_t numOfSupportVnodes; int16_t numOfCommitThreads; int8_t enableTelem; int32_t statusInterval; diff --git a/include/os/osEnv.h b/include/os/osEnv.h index b857670214..a7fd86776c 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -21,7 +21,6 @@ extern "C" { #endif extern char tsOsName[]; -extern char tsDnodeDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 910d05e9a3..9ddadc9ba6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,17 +15,18 @@ #define _DEFAULT_SOURCE #include "os.h" + #include "taosdef.h" #include "taoserror.h" -#include "ulog.h" -#include "tlog.h" -#include "tconfig.h" -#include "tglobal.h" #include "tcompare.h" -#include "tutil.h" -#include "ttimezone.h" -#include "tlocale.h" +#include "tconfig.h" #include "tep.h" +#include "tglobal.h" +#include "tlocale.h" +#include "tlog.h" +#include "ttimezone.h" +#include "tutil.h" +#include "ulog.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -36,22 +37,24 @@ uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second int8_t tsEnableTelemetryReporting = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; +int32_t tsNumOfSupportVnodes = 16; // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; //disable this, means query, show command use udp protocol as default -int32_t tsMaxShellConns = 50000; +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; +int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default +int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; -int32_t tsShellActivityTimer = 3; // second +int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; -int8_t tsDaylight = 0; +int8_t tsDaylight = 0; int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -int64_t tsMaxVnodeQueuedBytes = 1024*1024*1024; //1GB - +int8_t tsEnableSlaveQuery = 1; +int8_t tsEnableAdjustMaster = 1; +int8_t tsPrintAuth = 0; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -79,7 +82,7 @@ int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; -int8_t tsTscEnableRecordSql = 0; +int8_t tsTscEnableRecordSql = 0; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -89,7 +92,7 @@ int32_t tsMaxNumOfOrderedResults = 100000; int32_t tsMinSlidingTime = 10; // the maxinum number of distict query result -int32_t tsMaxNumOfDistinctResults = 1000 * 10000; +int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 us for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; @@ -101,7 +104,7 @@ int32_t tsMaxStreamComputDelay = 20000; int32_t tsStreamCompStartDelay = 10000; // the stream computing delay time after executing failed, change accordingly -int32_t tsRetryStreamCompDelay = 10*1000; +int32_t tsRetryStreamCompDelay = 10 * 1000; // The delayed computing ration. 10% of the whole computing time window by default. float tsStreamComputDelayRatio = 0.1f; @@ -120,30 +123,16 @@ int64_t tsQueryBufferSizeBytes = -1; int32_t tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int8_t tsKeepOriginalColumnName = 0; +int8_t tsKeepOriginalColumnName = 0; -// tsdb config +// long query death-lock +int8_t tsDeadLockKillQuery = 0; + +// tsdb config // For backward compatibility bool tsdbForceKeepFile = false; -// balance -int8_t tsEnableFlowCtrl = 1; -int8_t tsEnableSlaveQuery = 1; -int8_t tsEnableAdjustMaster = 1; - - -// monitor -char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; -char tsInternalPass[] = "secretkey"; - -// internal -int8_t tsCompactMnodeWal = 0; -int8_t tsPrintAuth = 0; -char tsVnodeDir[PATH_MAX] = {0}; -char tsDnodeDir[PATH_MAX] = {0}; -char tsMnodeDir[PATH_MAX] = {0}; - -int32_t tsDiskCfgNum = 0; +int32_t tsDiskCfgNum = 0; #ifndef _STORAGE SDiskCfg tsDiskCfg[1]; @@ -160,31 +149,28 @@ SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; -int32_t tsTotalMemoryMB = 0; +float tsTotalTmpDirGB = 0; +float tsTotalDataDirGB = 0; +float tsAvailTmpDirectorySpace = 0; +float tsAvailDataDirGB = 0; +float tsUsedDataDirGB = 0; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 2.0f; +int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; -#ifdef TD_TSZ // // lossy compress 6 // -char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty can close lossy compress. -// below option can take effect when tsLossyColumns not empty -double fPrecision = 1E-8; // float column precision -double dPrecision = 1E-16; // double column precision -uint32_t maxRange = 500; // max range -uint32_t curRange = 100; // range -char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR -#endif +char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty + // can close lossy compress. +// below option can take effect when tsLossyColumns not empty +double tsFPrecision = 1E-8; // float column precision +double tsDPrecision = 1E-16; // double column precision +uint32_t tsMaxRange = 500; // max range +uint32_t tsCurRange = 100; // range +char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR -// long query death-lock -int8_t tsDeadLockKillQuery = 0; int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; @@ -195,13 +181,12 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { - if (debugFlag != 0) { + if (debugFlag != 0) { mDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; jniDebugFlag = debugFlag; - odbcDebugFlag = debugFlag; - qDebugFlag = debugFlag; + qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; sDebugFlag = debugFlag; @@ -213,12 +198,12 @@ void taosSetAllDebugFlag() { } int32_t taosCfgDynamicOptions(char *msg) { - char *option, *value; - int32_t olen, vlen; - int32_t vint = 0; + char *option, *value; + int32_t olen, vlen; + int32_t vint = 0; paGetToken(msg, &option, &olen); - if (olen == 0) return -1;; + if (olen == 0) return -1; paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) @@ -231,9 +216,9 @@ int32_t taosCfgDynamicOptions(char *msg) { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; - //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; + // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; - + int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; @@ -262,7 +247,7 @@ int32_t taosCfgDynamicOptions(char *msg) { return 0; } if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); + taosSetAllDebugFlag(); } return 0; } @@ -323,7 +308,7 @@ static void doInitGlobalConfig(void) { srand(taosSafeRand()); SGlobalCfg cfg = {0}; - + // ip address cfg.option = "firstEp"; cfg.ptr = tsFirst; @@ -366,6 +351,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); + cfg.option = "supportVnodes"; + cfg.ptr = &tsNumOfSupportVnodes; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 65536; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosAddConfigOption(cfg); + // directory cfg.option = "configDir"; cfg.ptr = configDir; @@ -442,8 +437,8 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsMaxNumOfDistinctResults; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10*10000; - cfg.maxValue = 10000*10000; + cfg.minValue = 10 * 10000; + cfg.maxValue = 10000 * 10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); @@ -749,17 +744,6 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - // module configs - cfg.option = "flowctrl"; - cfg.ptr = &tsEnableFlowCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); cfg.option = "slaveQuery"; @@ -893,16 +877,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "odbcDebugFlag"; - cfg.ptr = &odbcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "uDebugFlag"; cfg.ptr = &uDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1034,7 +1008,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // enable kill long query + // enable kill long query cfg.option = "deadLockKillQuery"; cfg.ptr = &tsDeadLockKillQuery; cfg.valType = TAOS_CFG_VTYPE_INT8; @@ -1066,7 +1040,6 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "dPrecision"; @@ -1100,23 +1073,20 @@ static void doInitGlobalConfig(void) { taosAddConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else - //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); + // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif - } -void taosInitGlobalCfg() { - pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); -} +void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } int32_t taosCheckAndPrintCfg() { - char fqdn[TSDB_FQDN_LEN]; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); } - + if (tsLocalFqdn[0] == 0) { taosGetFqdn(tsLocalFqdn); } @@ -1143,7 +1113,7 @@ int32_t taosCheckAndPrintCfg() { if (taosDirExist(tsTempDir) != 0) { return -1; } - + taosGetSystemInfo(); tsSetLocale(); diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 70dca0e4df..8161b8d125 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -139,7 +139,7 @@ void dmnWaitSignal() { void dmnInitOption(SDnodeOpt *pOption) { pOption->sver = 30000000; //3.0.0.0 pOption->numOfCores = tsNumOfCores; - pOption->numOfSupportVnodes = 16; + pOption->numOfSupportVnodes = tsNumOfSupportVnodes; pOption->numOfCommitThreads = 1; pOption->statusInterval = tsStatusInterval; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 8807579564..8a50449a61 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -370,8 +370,8 @@ void dndSendStatusMsg(SDnode *pDnode) { pStatus->clusterId = htobe64(pMgmt->clusterId); pStatus->rebootTime = htobe64(pMgmt->rebootTime); pStatus->updateTime = htobe64(pMgmt->updateTime); - pStatus->numOfCores = htons(pDnode->opt.numOfCores); - pStatus->numOfSupportVnodes = htons(pDnode->opt.numOfSupportVnodes); + pStatus->numOfCores = htonl(pDnode->opt.numOfCores); + pStatus->numOfSupportVnodes = htonl(pDnode->opt.numOfSupportVnodes); tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN); pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval); diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 96bb6e5390..54d7e73be6 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -57,7 +57,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); - CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "support_vnodes"); CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); diff --git a/source/dnode/mgmt/impl/test/sut/src/base.cpp b/source/dnode/mgmt/impl/test/sut/src/base.cpp index 98371e9893..e14dc94d31 100644 --- a/source/dnode/mgmt/impl/test/sut/src/base.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/base.cpp @@ -24,7 +24,6 @@ void Testbase::InitLog(const char* path) { tmrDebugFlag = 0; uDebugFlag = 143; rpcDebugFlag = 0; - odbcDebugFlag = 0; qDebugFlag = 0; wDebugFlag = 0; sDebugFlag = 0; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b93fc0951c..669b97e54d 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -125,8 +125,8 @@ typedef struct { int64_t lastAccessTime; int32_t accessTimes; int16_t numOfVnodes; - int16_t numOfSupportVnodes; - int16_t numOfCores; + int32_t numOfSupportVnodes; + int32_t numOfCores; EDndStatus status; EDndReason offlineReason; uint16_t port; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 43b458a52a..d101be74cc 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -278,8 +278,8 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->clusterId = htobe64(pStatus->clusterId); pStatus->rebootTime = htobe64(pStatus->rebootTime); pStatus->updateTime = htobe64(pStatus->updateTime); - pStatus->numOfCores = htons(pStatus->numOfCores); - pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes); + pStatus->numOfCores = htonl(pStatus->numOfCores); + pStatus->numOfSupportVnodes = htonl(pStatus->numOfSupportVnodes); pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval); pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } @@ -638,7 +638,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "max_vnodes"); + strcpy(pSchema[cols].name, "support_vnodes"); pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 23b62d2dfb..e787d351a0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -265,7 +265,7 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) { taosArrayPush(pArray, pDnode); } - mDebug("dnode:%d, numOfVnodes:%d numOfSupportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes, + mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes, pDnode->numOfSupportVnodes, isMnode, isReady); sdbRelease(pSdb, pDnode); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 7e7f673943..95f2fe76e6 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -85,7 +85,6 @@ int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t odbcDebugFlag = 131; int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 0ccc85c347..fcc11ca213 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -120,6 +120,7 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG +echo "supportVnodes 16" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG diff --git a/tests/script/tmp/dnodes.sim b/tests/script/tmp/dnodes.sim index f13f6026f9..9252ac50bb 100644 --- a/tests/script/tmp/dnodes.sim +++ b/tests/script/tmp/dnodes.sim @@ -1,63 +1,81 @@ -system sh/stop_dnodes.sh - - ############## config parameter ##################### -$node1 = 192.168.0.201 +$node1 = 192.168.101.12 $node2 = 192.168.0.202 $node3 = 192.168.0.203 $node4 = 192.168.0.204 $self = $node1 -$num = 25 +$num = 5 -############### deploy firstEp ##################### +#deploy = 0, start = 1, stop = 2 +$option = 0 +print =============== option:$option + + +############### stop dnodes ##################### +if $option == 0 then + system sh/stop_dnodes.sh +endi + +############### process firstEp ##################### $firstEp = $node1 . :7100 $firstPort = 7100 if $self == $node1 then - system sh/deploy.sh -n dnode1 -i 1 - system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp - system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp - system sh/cfg.sh -n dnode1 -c fqdn -v $node1 - system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort + if $option == 1 then + system sh/exec.sh -n dnode1 -s start + endi - system sh/exec.sh -n dnode1 -s start - sql connect + if $option == 2 then + system sh/exec.sh -n dnode1 -s stop -x SIGINT + endi + + if $option == 0 then + system sh/deploy.sh -n dnode1 -i 1 + system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp + system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp + system sh/cfg.sh -n dnode1 -c fqdn -v $node1 + system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort + system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node1 port $port - endw + system sh/exec.sh -n dnode1 -s start + sql connect - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node2 port $port - endw + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node1 port $port + endw - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node3 port $port - endw + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node2 port $port + endw - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node4 port $port - endw + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node3 port $port + endw + + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node4 port $port + endw + endi endi -############### deploy nodes ##################### +############### process nodes ##################### $i = 0 while $i < $num @@ -67,11 +85,21 @@ while $i < $num $dnodename = dnode . $index $i = $i + 1 - system sh/deploy.sh -n $dnodename -i 1 - system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp - system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp - system sh/cfg.sh -n $dnodename -c fqdn -v $self - system sh/cfg.sh -n $dnodename -c serverPort -v $port + if $option == 1 then + system sh/exec.sh -n $dnodename -s start + endi - system sh/exec.sh -n $dnodename -s start + if $option == 2 then + system sh/exec.sh -n $dnodename -s stop -x SIGINT + endi + + if $option == 0 then + system sh/deploy.sh -n $dnodename -i 1 + system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp + system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp + system sh/cfg.sh -n $dnodename -c fqdn -v $self + system sh/cfg.sh -n $dnodename -c serverPort -v $port + + system sh/exec.sh -n $dnodename -s start + endi endw From 70df7a46bc8f9c894a19a42a965557962a28f22e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 19:40:31 -0800 Subject: [PATCH 26/97] minor changes --- tests/script/tmp/dnodes.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tmp/dnodes.sim b/tests/script/tmp/dnodes.sim index 9252ac50bb..cc7d1090f1 100644 --- a/tests/script/tmp/dnodes.sim +++ b/tests/script/tmp/dnodes.sim @@ -1,5 +1,5 @@ ############## config parameter ##################### -$node1 = 192.168.101.12 +$node1 = 192.168.0.201 $node2 = 192.168.0.202 $node3 = 192.168.0.203 $node4 = 192.168.0.204 From 53a2e752d17c7cb23fe1a590275514f566b0cb15 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 19:42:18 -0800 Subject: [PATCH 27/97] minor changes --- tests/script/tmp/dnodes.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/tmp/dnodes.sim b/tests/script/tmp/dnodes.sim index cc7d1090f1..a3f9a0c173 100644 --- a/tests/script/tmp/dnodes.sim +++ b/tests/script/tmp/dnodes.sim @@ -5,7 +5,7 @@ $node3 = 192.168.0.203 $node4 = 192.168.0.204 $self = $node1 -$num = 5 +$num = 25 #deploy = 0, start = 1, stop = 2 $option = 0 From 68f8d6451efe87db5e994d66df3d83a2d958011e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 19:49:23 -0800 Subject: [PATCH 28/97] The minimum value of vgroupid is set to 2 --- source/dnode/mnode/impl/src/mndVgroup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e787d351a0..3cb5173967 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -333,6 +333,8 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { uint32_t hashMax = UINT32_MAX; uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups; + if (maxVgId < 2) maxVgId = 2; + for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) { SVgObj *pVgroup = &pVgroups[v]; pVgroup->vgId = maxVgId++; From 5f9826b93a34b39294bcfc865dd3f4e50708ef7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 21:23:52 -0800 Subject: [PATCH 29/97] minor changes --- source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 30 +++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index b726743c9a..5b0dc482ff 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -103,7 +103,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TRANS)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TRANS_RSP)] = dndProcessMnodeWriteMsg; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d101be74cc..841a7696f3 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -287,15 +287,17 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SStatusMsg *pStatus = pMsg->rpcMsg.pCont; + SDnodeObj *pDnode = NULL; + int32_t code = -1; + mndParseStatusMsg(pStatus); - SDnodeObj *pDnode = NULL; if (pStatus->dnodeId == 0) { pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp); if (pDnode == NULL) { mDebug("dnode:%s, not created yet", pStatus->dnodeEp); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto PROCESS_STATUS_MSG_OVER; } } else { pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId); @@ -305,9 +307,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode->offlineReason = DND_REASON_DNODE_ID_NOT_MATCH; } mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); - mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto PROCESS_STATUS_MSG_OVER; } } @@ -315,10 +316,9 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } - mndReleaseDnode(pMnode, pDnode); mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; - return -1; + goto PROCESS_STATUS_MSG_OVER; } if (pStatus->dnodeId == 0) { @@ -330,9 +330,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId, pMnode->clusterId); - mndReleaseDnode(pMnode, pDnode); terrno != TSDB_CODE_MND_INVALID_CLUSTER_ID; - return -1; + goto PROCESS_STATUS_MSG_OVER; } else { pDnode->accessTimes++; mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); @@ -345,9 +344,8 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { if (0 != ret) { pDnode->offlineReason = ret; mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[ret]); - mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG; - return -1; + goto PROCESS_STATUS_MSG_OVER; } mInfo("dnode:%d, from offline to online", pDnode->id); @@ -363,20 +361,22 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { int32_t contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp); SStatusRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + goto PROCESS_STATUS_MSG_OVER; } pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); - mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); + // mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); pMsg->contLen = contLen; pMsg->pCont = pRsp; - mndReleaseDnode(pMnode, pDnode); - return 0; + code = 0; + +PROCESS_STATUS_MSG_OVER: + mndReleaseDnode(pMnode, pDnode); + return code; } static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { From 28359096b345b7de97c490ef68b36984d5dfe5ba Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 21:43:34 -0800 Subject: [PATCH 30/97] dnode offline reason --- source/dnode/mnode/impl/inc/mndDef.h | 8 ------- source/dnode/mnode/impl/inc/mndDnode.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 28 +++++++++++-------------- source/dnode/mnode/impl/src/mndVgroup.c | 8 +++---- 4 files changed, 17 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 669b97e54d..338024c20a 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -74,13 +74,6 @@ typedef enum { typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; -typedef enum { - DND_STATUS_OFFLINE = 0, - DND_STATUS_READY = 1, - DND_STATUS_CREATING = 2, - DND_STATUS_DROPPING = 3 -} EDndStatus; - typedef enum { DND_REASON_ONLINE = 0, DND_REASON_STATUS_MSG_TIMEOUT, @@ -127,7 +120,6 @@ typedef struct { int16_t numOfVnodes; int32_t numOfSupportVnodes; int32_t numOfCores; - EDndStatus status; EDndReason offlineReason; uint16_t port; char fqdn[TSDB_FQDN_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 764dfbffc1..10b37c0888 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,7 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); -bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode); +bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 841a7696f3..bcd18177a9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -40,8 +40,6 @@ static const char *offlineReason[] = { "unknown", }; -static const char *dnodeStatus[] = {"offline", "ready", "creating", "dropping"}; - static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -208,10 +206,13 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } -bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { +bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode) { int64_t ms = taosGetTimestampMs(); int64_t interval = ABS(pDnode->lastAccessTime - ms); if (interval > 3500 * pMnode->cfg.statusInterval) { + if (pDnode->rebootTime > 0) { + pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; + } return false; } return true; @@ -303,7 +304,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId); if (pDnode == NULL) { pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp); - if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { + if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_DNODE_ID_NOT_MATCH; } mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); @@ -313,7 +314,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } if (pStatus->sver != pMnode->cfg.sver) { - if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { + if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); @@ -325,12 +326,12 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { mDebug("dnode:%d %s, first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); } else { if (pStatus->clusterId != pMnode->clusterId) { - if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { + if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_CLUSTER_ID_NOT_MATCH; } mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId, pMnode->clusterId); - terrno != TSDB_CODE_MND_INVALID_CLUSTER_ID; + terrno = TSDB_CODE_MND_INVALID_CLUSTER_ID; goto PROCESS_STATUS_MSG_OVER; } else { pDnode->accessTimes++; @@ -338,7 +339,7 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } } - if (pDnode->status == DND_STATUS_OFFLINE) { + if (/*pDnode->status == DND_STATUS_OFFLINE*/1) { // Verify whether the cluster parameters are consistent when status change from offline to ready int32_t ret = mndCheckClusterCfgPara(pMnode, &pStatus->clusterCfg); if (0 != ret) { @@ -355,7 +356,6 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { pDnode->numOfCores = pStatus->numOfCores; pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; pDnode->lastAccessTime = taosGetTimestampMs(); - pDnode->status = DND_STATUS_READY; int32_t numOfEps = mndGetDnodeSize(pMnode); int32_t contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp); @@ -686,6 +686,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); if (pShow->pIter == NULL) break; + bool online = mndIsDnodeOnline(pMnode, pDnode); cols = 0; @@ -706,8 +707,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - const char *status = dnodeStatus[pDnode->status]; - STR_TO_VARSTR(pWrite, status); + STR_TO_VARSTR(pWrite, online ? "ready" : "offline"); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -715,11 +715,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pDnode->status == DND_STATUS_READY) { - STR_TO_VARSTR(pWrite, ""); - } else { - STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); - } + STR_TO_VARSTR(pWrite, online ? "" : offlineReason[pDnode->offlineReason]); cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 3cb5173967..6478cdaec9 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -260,13 +260,13 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) { pDnode->numOfVnodes++; } - bool isReady = mndIsDnodeInReadyStatus(pMnode, pDnode); - if (isReady) { + bool online = mndIsDnodeOnline(pMnode, pDnode); + if (online) { taosArrayPush(pArray, pDnode); } - mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes, - pDnode->numOfSupportVnodes, isMnode, isReady); + mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d", pDnode->id, numOfVnodes, + pDnode->numOfSupportVnodes, isMnode, online); sdbRelease(pSdb, pDnode); } From 212376551d3fb88bc6a92475ba47ab380165164e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 05:43:44 +0000 Subject: [PATCH 31/97] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 52 ++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 4ef7dc01a4..d76ee369bf 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -38,6 +38,11 @@ struct SMetaDB { DB_ENV *pEvn; }; +typedef struct { + int32_t nCols; + SSchema *pSchema; +} SSchemaWrapper; + typedef int (*bdbIdxCbPtr)(DB *, const DBT *, const DBT *, DBT *); static SMetaDB *metaNewDB(); @@ -55,6 +60,7 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT * static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); +static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -432,6 +438,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { /* ------------------------ FOR QUERY ------------------------ */ int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { +#if 0 DBT key = {0}; DBT value = {0}; SMetaDB * pMetaDB = pMeta->pDB; @@ -490,7 +497,7 @@ int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { pMsg->tuid = tbCfg.stbCfg.suid; memcpy(pMsg->pSchema, tbCfg.stbCfg.pSchema, sizeof(SSchema) * tbCfg.stbCfg.nCols); memcpy(POINTER_SHIFT(pMsg->pSchema, sizeof(SSchema) * tbCfg.stbCfg.nCols), tbCfg.stbCfg.pTagSchema, - sizeof(SSchema) * tbCfg.stbCfg.nTagCols); + sizeof(SSchema) * tbCfg.stbCfg.nTagCols); break; case META_CHILD_TABLE: ASSERT(0); @@ -505,5 +512,48 @@ int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { *ppMsg = pMsg; +#endif return 0; +} + +static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { + uint32_t nCols; + SSchemaWrapper *pSW = NULL; + SMetaDB * pDB = pMeta->pDB; + int ret; + void * pBuf; + SSchema * pSchema; + SSchemaKey schemaKey = {uid, sver}; + DBT key = {0}; + DBT value = {0}; + + // Set key/value properties + key.data = &schemaKey; + key.size = sizeof(schemaKey); + + // Query + ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0); + if (ret != 0) { + return NULL; + } + + // Decode the schema + pBuf = value.data; + taosDecodeFixedI32(&pBuf, &nCols); + pSW = (SSchemaWrapper *)malloc(sizeof(*pSW) + sizeof(SSchema) * nCols); + if (pSW == NULL) { + return NULL; + } + + pSW->pSchema = POINTER_SHIFT(pSW, sizeof(*pSW)); + + for (int i = 0; i < nCols; i++) { + pSchema = pSW->pSchema + i; + taosDecodeFixedI8(&pBuf, &(pSchema->type)); + taosDecodeFixedI32(&pBuf, &(pSchema->colId)); + taosDecodeFixedI32(&pBuf, &(pSchema->bytes)); + taosDecodeStringTo(&pBuf, pSchema->name); + } + + return pSW; } \ No newline at end of file From 62f620fa65f7df048446a59b62a9bf1c25a7ca0d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 22:22:09 -0800 Subject: [PATCH 32/97] process the status msg only once --- source/dnode/mgmt/impl/src/dndDnode.c | 26 +++---- source/dnode/mnode/impl/inc/mndDnode.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 94 +++++++++++++------------ source/dnode/mnode/impl/src/mndVgroup.c | 3 +- 4 files changed, 66 insertions(+), 59 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 8a50449a61..a057ac36b5 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -397,7 +397,7 @@ void dndSendStatusMsg(SDnode *pDnode) { static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:% " PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosWLockLatch(&pMgmt->latch); pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; @@ -440,19 +440,21 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { } SStatusRsp *pRsp = pMsg->pCont; - SDnodeCfg *pCfg = &pRsp->dnodeCfg; - pCfg->dnodeId = htonl(pCfg->dnodeId); - pCfg->clusterId = htobe64(pCfg->clusterId); - dndUpdateDnodeCfg(pDnode, pCfg); + if (pMsg->pCont != NULL) { + SDnodeCfg *pCfg = &pRsp->dnodeCfg; + pCfg->dnodeId = htonl(pCfg->dnodeId); + pCfg->clusterId = htobe64(pCfg->clusterId); + dndUpdateDnodeCfg(pDnode, pCfg); - SDnodeEps *pDnodeEps = &pRsp->dnodeEps; - pDnodeEps->num = htonl(pDnodeEps->num); - for (int32_t i = 0; i < pDnodeEps->num; ++i) { - pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); - pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); + SDnodeEps *pDnodeEps = &pRsp->dnodeEps; + pDnodeEps->num = htonl(pDnodeEps->num); + for (int32_t i = 0; i < pDnodeEps->num; ++i) { + pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); + pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); + } + + dndUpdateDnodeEps(pDnode, pDnodeEps); } - - dndUpdateDnodeEps(pDnode, pDnodeEps); pMgmt->statusSent = 0; } diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 10b37c0888..c76186c0a2 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,7 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); -bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode); +bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index bcd18177a9..ca20ba61a1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -206,9 +206,8 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } -bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode) { - int64_t ms = taosGetTimestampMs(); - int64_t interval = ABS(pDnode->lastAccessTime - ms); +bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { + int64_t interval = ABS(pDnode->lastAccessTime - curMs); if (interval > 3500 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; @@ -313,33 +312,37 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } } - if (pStatus->sver != pMnode->cfg.sver) { - if (pDnode != NULL) { - pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; - } - mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); - terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; - goto PROCESS_STATUS_MSG_OVER; - } + int64_t curMs = taosGetTimestampMs(); + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool needCheckCfg = !(online && pDnode->rebootTime == pStatus->rebootTime); - if (pStatus->dnodeId == 0) { - mDebug("dnode:%d %s, first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); - } else { - if (pStatus->clusterId != pMnode->clusterId) { + if (needCheckCfg) { + if (pStatus->sver != pMnode->cfg.sver) { if (pDnode != NULL) { - pDnode->offlineReason = DND_REASON_CLUSTER_ID_NOT_MATCH; + pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } - mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId, - pMnode->clusterId); - terrno = TSDB_CODE_MND_INVALID_CLUSTER_ID; + mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); + terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; goto PROCESS_STATUS_MSG_OVER; - } else { - pDnode->accessTimes++; - mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } - } - if (/*pDnode->status == DND_STATUS_OFFLINE*/1) { + if (pStatus->dnodeId == 0) { + mDebug("dnode:%d %s, first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); + } else { + if (pStatus->clusterId != pMnode->clusterId) { + if (pDnode != NULL) { + pDnode->offlineReason = DND_REASON_CLUSTER_ID_NOT_MATCH; + } + mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId, + pMnode->clusterId); + terrno = TSDB_CODE_MND_INVALID_CLUSTER_ID; + goto PROCESS_STATUS_MSG_OVER; + } else { + pDnode->accessTimes++; + mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); + } + } + // Verify whether the cluster parameters are consistent when status change from offline to ready int32_t ret = mndCheckClusterCfgPara(pMnode, &pStatus->clusterCfg); if (0 != ret) { @@ -350,28 +353,28 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { } mInfo("dnode:%d, from offline to online", pDnode->id); + + pDnode->rebootTime = pStatus->rebootTime; + pDnode->numOfCores = pStatus->numOfCores; + pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; + + int32_t numOfEps = mndGetDnodeSize(pMnode); + int32_t contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp); + SStatusRsp *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto PROCESS_STATUS_MSG_OVER; + } + + pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); + pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); + mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); + + pMsg->contLen = contLen; + pMsg->pCont = pRsp; } - pDnode->rebootTime = pStatus->rebootTime; - pDnode->numOfCores = pStatus->numOfCores; - pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; - pDnode->lastAccessTime = taosGetTimestampMs(); - - int32_t numOfEps = mndGetDnodeSize(pMnode); - int32_t contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp); - SStatusRsp *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto PROCESS_STATUS_MSG_OVER; - } - - pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); - pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); - // mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); - - pMsg->contLen = contLen; - pMsg->pCont = pRsp; - + pDnode->lastAccessTime = curMs; code = 0; PROCESS_STATUS_MSG_OVER: @@ -682,11 +685,12 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i int32_t cols = 0; SDnodeObj *pDnode = NULL; char *pWrite; + int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); if (pShow->pIter == NULL) break; - bool online = mndIsDnodeOnline(pMnode, pDnode); + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); cols = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 6478cdaec9..1b0026cd13 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -260,7 +260,8 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) { pDnode->numOfVnodes++; } - bool online = mndIsDnodeOnline(pMnode, pDnode); + int64_t curMs = taosGetTimestampMs(); + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); if (online) { taosArrayPush(pArray, pDnode); } From 71e2b6b3f227dbc09181f52f13c4e231deb387ff Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Dec 2021 01:22:19 -0500 Subject: [PATCH 33/97] TD-12506 insert bug fix --- source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 3 + source/client/src/clientImpl.c | 10 +- source/libs/parser/src/insertParser.c | 261 ------------------ source/libs/parser/src/parserUtil.c | 22 +- .../libs/parser/test/mockCatalogService.cpp | 5 +- source/libs/planner/src/physicalPlan.c | 4 + source/libs/scheduler/src/scheduler.c | 2 +- 8 files changed, 32 insertions(+), 276 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 021f650254..706e838b94 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -121,6 +121,7 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; + uint64_t affectedRows; SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index db1ea435f1..960ba95324 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -18,6 +18,7 @@ #include "clientInt.h" #include "clientLog.h" #include "query.h" +#include "scheduler.h" #include "tmsg.h" #include "tcache.h" #include "tconfig.h" @@ -230,6 +231,8 @@ void taos_init_imp(void) { SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; catalogInit(&cfg); + SSchedulerCfg scfg = {.maxJobNum = 100}; + schedulerInit(&scfg); tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); taosSetCoreDump(true); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3d295830c7..68d448039d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -196,7 +196,15 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { return TSDB_CODE_SUCCESS; } +int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQuery, SQueryDag** pDag) { + pRequest->type = pQuery->type; + return qCreateQueryDag(pQuery, pDag); +} + int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { + if (TSDB_SQL_INSERT == pRequest->type) { + return scheduleExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob, &pRequest->affectedRows); + } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); } @@ -283,7 +291,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQuery)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); } else { - CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return); + CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 991bde5ed2..66966f75db 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -64,64 +64,6 @@ typedef struct SInsertParseContext { SInsertStmtInfo* pOutput; } SInsertParseContext; -static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) { - errno = 0; - *value = strtold(pToken->z, endPtr); - - // not a valid integer number, return error - if ((*endPtr - pToken->z) != pToken->n) { - return TK_ILLEGAL; - } - - return pToken->type; -} - -static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { - errno = 0; - int32_t ret = 0; - - char* endPtr = NULL; - if (type == TK_FLOAT) { - double v = strtod(z, &endPtr); - if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { - ret = -1; - } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { - ret = -1; - } else { - *value = (int64_t) round(v); - } - - errno = 0; - return ret; - } - - int32_t radix = 10; - if (type == TK_HEX) { - radix = 16; - } else if (type == TK_BIN) { - radix = 2; - } - - // the string may be overflow according to errno - if (!issigned) { - const char *p = z; - while(*p != 0 && *p == ' ') p++; - if (*p != 0 && *p == '-') { return -1;} - - *value = strtoull(z, &endPtr, radix); - } else { - *value = strtoll(z, &endPtr, radix); - } - - // not a valid integer number, return error - if (endPtr - z != n || errno == ERANGE) { - ret = -1; - } - - errno = 0; - return ret; -} - static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -159,10 +101,8 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char tableName[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&name, tableName); - SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx; CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); - SVgroupInfo vg; CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); @@ -349,207 +289,6 @@ static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *p return TSDB_CODE_SUCCESS; } -//static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { -// if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && -// type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || -// (pToken->n == 0) || (type == TK_RP)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); -// } -// -// if (IS_NUMERIC_TYPE(type) && pToken->n == 0) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); -// } -// -// // Remove quotation marks -// if (TK_STRING == type) { -// if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { -// return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); -// } -// -// // delete escape character: \\, \', \" -// char delim = pToken->z[0]; -// int32_t cnt = 0; -// int32_t j = 0; -// for (uint32_t k = 1; k < pToken->n - 1; ++k) { -// if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { -// tmpTokenBuf[j] = pToken->z[k + 1]; -// cnt++; -// j++; -// k++; -// continue; -// } -// tmpTokenBuf[j] = pToken->z[k]; -// j++; -// } -// -// tmpTokenBuf[j] = 0; -// pToken->z = tmpTokenBuf; -// pToken->n -= 2 + cnt; -// } -// -// return TSDB_CODE_SUCCESS; -//} - -//static FORCE_INLINE int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { -// int64_t iv; -// char *endptr = NULL; -// bool isSigned = false; -// -// CHECK_CODE(checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf)); -// -// if (isNullStr(pToken)) { -// if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { -// int64_t tmpVal = 0; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// return func(getNullValue(pSchema->type), 0, param); -// } -// -// switch (pSchema->type) { -// case TSDB_DATA_TYPE_BOOL: { -// if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { -// if (strncmp(pToken->z, "true", pToken->n) == 0) { -// return func(&TRUE_VALUE, pSchema->bytes, param); -// } else if (strncmp(pToken->z, "false", pToken->n) == 0) { -// return func(&FALSE_VALUE, pSchema->bytes, param); -// } else { -// return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); -// } -// } else if (pToken->type == TK_INTEGER) { -// return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); -// } else if (pToken->type == TK_FLOAT) { -// return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); -// } else { -// return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); -// } -// } -// -// case TSDB_DATA_TYPE_TINYINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); -// } else if (!IS_VALID_TINYINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); -// } -// -// uint8_t tmpVal = (uint8_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_UTINYINT:{ -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); -// } else if (!IS_VALID_UTINYINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); -// } -// uint8_t tmpVal = (uint8_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_SMALLINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); -// } else if (!IS_VALID_SMALLINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); -// } -// int16_t tmpVal = (int16_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_USMALLINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); -// } else if (!IS_VALID_USMALLINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); -// } -// uint16_t tmpVal = (uint16_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_INT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); -// } else if (!IS_VALID_INT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); -// } -// int32_t tmpVal = (int32_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_UINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); -// } else if (!IS_VALID_UINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); -// } -// uint32_t tmpVal = (uint32_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_BIGINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); -// } else if (!IS_VALID_BIGINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); -// } -// return func(&iv, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_UBIGINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); -// } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); -// } -// uint64_t tmpVal = (uint64_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_FLOAT: { -// double dv; -// if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); -// } -// if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); -// } -// float tmpVal = (float)dv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_DOUBLE: { -// double dv; -// if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); -// } -// if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); -// } -// return func(&dv, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_BINARY: { -// // too long values will return invalid sql, not be truncated automatically -// if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { -// return buildSyntaxErrMsg(pMsgBuf, "string data overflow", pToken->z); -// } -// return func(pToken->z, pToken->n, param); -// } -// case TSDB_DATA_TYPE_NCHAR: { -// return func(pToken->z, pToken->n, param); -// } -// case TSDB_DATA_TYPE_TIMESTAMP: { -// int64_t tmpVal; -// if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); -// } -// return func(&tmpVal, pSchema->bytes, param); -// } -// } -// -// return TSDB_CODE_FAILED; -//} - // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { int32_t nCols = pColList->numOfCols; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 20f330247e..79c6cbb655 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1649,9 +1649,9 @@ static bool isNullStr(SToken *pToken) { } static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { - if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || - (pToken->n == 0) || (type == TK_RP)) { + if ((pToken->type != TK_NOW && pToken->type != TK_INTEGER && pToken->type != TK_STRING && pToken->type != TK_FLOAT && pToken->type != TK_BOOL && + pToken->type != TK_NULL && pToken->type != TK_HEX && pToken->type != TK_OCT && pToken->type != TK_BIN) || + (pToken->n == 0) || (pToken->type == TK_RP)) { return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } @@ -1795,7 +1795,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); @@ -1806,7 +1806,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_UTINYINT:{ - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (!IS_VALID_UTINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); @@ -1816,7 +1816,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); @@ -1826,7 +1826,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (!IS_VALID_USMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); @@ -1836,7 +1836,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); @@ -1846,7 +1846,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (!IS_VALID_UINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); @@ -1856,7 +1856,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } else if (!IS_VALID_BIGINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); @@ -1865,7 +1865,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index a651e6c1df..3be358fec8 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -97,8 +97,8 @@ public: int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(pTableName, db); const char* tname = tNameGetTableName(pTableName); int32_t code = copyTableSchemaMeta(db, tname, &table); @@ -111,6 +111,7 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo + vgInfo->vgId = 1; return 0; } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 8388458b4c..97c9cec7c7 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -207,6 +207,7 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { } taosArrayPush(currentLevel, &subplan); pCxt->pCurrentSubplan = subplan; + ++(pCxt->pDag->numOfSubplans); return subplan; } @@ -293,11 +294,14 @@ static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SArray* vgs = (SArray*)pPlanNode->pExtInfo; size_t numOfVg = taosArrayGetSize(vgs); for (int32_t i = 0; i < numOfVg; ++i) { + STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(vgs, i); vgroupInfoToEpSet(&blocks->vg, &subplan->execEpSet); subplan->pNode = NULL; subplan->pDataSink = createDataInserter(pCxt, blocks); + subplan->type = QUERY_TYPE_MODIFY; + RECOVERY_CURRENT_SUBPLAN(pCxt); } } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 7bd2205e43..12c07d69ee 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -166,7 +166,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) { } for (int32_t n = 0; n < levelPlanNum; ++n) { - SSubplan *plan = taosArrayGet(levelPlans, n); + SSubplan *plan = taosArrayGetP(levelPlans, n); SSchTask task = {0}; if (plan->type == QUERY_TYPE_MODIFY) { From 25f6ffd18db8fcb67b871095b9d043eac19d72c4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 22:22:34 -0800 Subject: [PATCH 34/97] crash --- source/dnode/mgmt/impl/src/dndDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index a057ac36b5..30b069f349 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -440,7 +440,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { } SStatusRsp *pRsp = pMsg->pCont; - if (pMsg->pCont != NULL) { + if (pMsg->pCont != NULL && pMsg->contLen != 0) { SDnodeCfg *pCfg = &pRsp->dnodeCfg; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->clusterId = htobe64(pCfg->clusterId); From a4cb6467c2e4ea5c00ce7f61cae30ef7ca0fa77a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 06:28:58 +0000 Subject: [PATCH 35/97] more progress --- include/dnode/vnode/meta/meta.h | 2 +- source/dnode/vnode/impl/src/vnodeQuery.c | 2 + source/dnode/vnode/meta/src/metaBDBImpl.c | 173 ++++++++++++---------- 3 files changed, 98 insertions(+), 79 deletions(-) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index cc5eabf3bf..74a0f85932 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -48,7 +48,7 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // For Query -int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg); +STbCfg *metaGetTableInfo(SMeta *pMeta, char *tbname); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 1c6924040c..a0bdfed6fb 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -45,6 +45,7 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +#if 0 STableInfoMsg *pReq = (STableInfoMsg *)(pMsg->pCont); STableMetaMsg *pRspMsg; int ret; @@ -63,5 +64,6 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO (*pRsp)->pCont = pRspMsg; +#endif return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index d76ee369bf..e70dcff7ad 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -60,7 +60,9 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT * static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); -static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver); +static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); +static STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +static STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -437,86 +439,89 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { } /* ------------------------ FOR QUERY ------------------------ */ -int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { -#if 0 - DBT key = {0}; - DBT value = {0}; - SMetaDB * pMetaDB = pMeta->pDB; - int ret; - STbCfg tbCfg; - SSchemaKey schemaKey; - DBT key1 = {0}; - DBT value1 = {0}; - uint32_t ncols; - void * pBuf; - int tlen; - STableMetaMsg *pMsg; - SSchema * pSchema; +STbCfg *metaGetTableInfo(SMeta *pMeta, char *tbname) { + STbCfg * pTbCfg = NULL; + STbCfg * pStbCfg = NULL; + tb_uid_t uid; + int32_t sver = 0; + SSchemaWrapper *pSW; - key.data = tbname; - key.size = strlen(tbname) + 1; - - ret = pMetaDB->pNameIdx->get(pMetaDB->pNameIdx, NULL, &key, &value, 0); - if (ret != 0) { - // TODO - return -1; + pTbCfg = metaGetTbInfoByName(pMeta, tbname, &uid); + if (pTbCfg == NULL) { + return NULL; } - metaDecodeTbInfo(value.data, &tbCfg); - - switch (tbCfg.type) { - case META_SUPER_TABLE: - schemaKey.uid = tbCfg.stbCfg.suid; - schemaKey.sver = 0; - - key1.data = &schemaKey; - key1.size = sizeof(schemaKey); - - ret = pMetaDB->pSchemaDB->get(pMetaDB->pSchemaDB, &key1, &value1, NULL, 0); - if (ret != 0) { - // TODO - return -1; - } - pBuf = value1.data; - pBuf = taosDecodeFixedU32(pBuf, &ncols); - - tlen = sizeof(STableMetaMsg) + (tbCfg.stbCfg.nTagCols + ncols) * sizeof(SSchema); - pMsg = calloc(1, tlen); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - strcpy(pMsg->tbFname, tbCfg.name); - pMsg->numOfTags = tbCfg.stbCfg.nTagCols; - pMsg->numOfColumns = ncols; - pMsg->tableType = tbCfg.type; - pMsg->sversion = 0; - pMsg->tversion = 0; - pMsg->suid = tbCfg.stbCfg.suid; - pMsg->tuid = tbCfg.stbCfg.suid; - memcpy(pMsg->pSchema, tbCfg.stbCfg.pSchema, sizeof(SSchema) * tbCfg.stbCfg.nCols); - memcpy(POINTER_SHIFT(pMsg->pSchema, sizeof(SSchema) * tbCfg.stbCfg.nCols), tbCfg.stbCfg.pTagSchema, - sizeof(SSchema) * tbCfg.stbCfg.nTagCols); - break; - case META_CHILD_TABLE: - ASSERT(0); - break; - case META_NORMAL_TABLE: - ASSERT(0); - break; - default: - ASSERT(0); - break; + if (pTbCfg->type == META_CHILD_TABLE) { + uid = pTbCfg->ctbCfg.suid; } - *ppMsg = pMsg; + pSW = metaGetTableSchema(pMeta, uid, 0, false); + if (pSW == NULL) { + return NULL; + } -#endif - return 0; + return pTbCfg; } -static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { +static STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { + STbCfg * pTbCfg = NULL; + SMetaDB *pDB = pMeta->pDB; + DBT key = {0}; + DBT value = {0}; + int ret; + + // Set key/value + key.data = &uid; + key.size = sizeof(uid); + + // Query + ret = pDB->pTbDB->get(pDB->pTbDB, NULL, &key, &value, 0); + if (ret != 0) { + return NULL; + } + + // Decode + pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + if (pTbCfg == NULL) { + return NULL; + } + + metaDecodeTbInfo(value.data, pTbCfg); + + return pTbCfg; +} + +static STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { + STbCfg * pTbCfg = NULL; + SMetaDB *pDB = pMeta->pDB; + DBT key = {0}; + DBT pkey = {0}; + DBT pvalue = {0}; + int ret; + + // Set key/value + key.data = tbname; + key.size = strlen(tbname); + + // Query + ret = pDB->pNameIdx->pget(pDB->pNameIdx, NULL, &key, &pkey, &pvalue, 0); + if (ret != 0) { + return NULL; + } + + // Decode + *uid = *(tb_uid_t *)(pkey.data); + pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + if (pTbCfg == NULL) { + return NULL; + } + + metaDecodeTbInfo(pvalue.data, pTbCfg); + + return pTbCfg; +} + +static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { uint32_t nCols; SSchemaWrapper *pSW = NULL; SMetaDB * pDB = pMeta->pDB; @@ -540,12 +545,24 @@ static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sv // Decode the schema pBuf = value.data; taosDecodeFixedI32(&pBuf, &nCols); - pSW = (SSchemaWrapper *)malloc(sizeof(*pSW) + sizeof(SSchema) * nCols); - if (pSW == NULL) { - return NULL; - } + if (isinline) { + pSW = (SSchemaWrapper *)malloc(sizeof(*pSW) + sizeof(SSchema) * nCols); + if (pSW == NULL) { + return NULL; + } + pSW->pSchema = POINTER_SHIFT(pSW, sizeof(*pSW)); + } else { + pSW = (SSchemaWrapper *)malloc(sizeof(*pSW)); + if (pSW == NULL) { + return NULL; + } - pSW->pSchema = POINTER_SHIFT(pSW, sizeof(*pSW)); + pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * nCols); + if (pSW->pSchema == NULL) { + free(pSW); + return NULL; + } + } for (int i = 0; i < nCols; i++) { pSchema = pSW->pSchema + i; From db2399fa3c2322aca40eccd96cffa4f637ef6feb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 07:05:49 +0000 Subject: [PATCH 36/97] more progress --- include/dnode/vnode/meta/meta.h | 9 ++- source/dnode/vnode/impl/inc/vnodeDef.h | 3 +- source/dnode/vnode/impl/src/vnodeQuery.c | 71 ++++++++++++++++++----- source/dnode/vnode/meta/src/metaBDBImpl.c | 38 +----------- 4 files changed, 71 insertions(+), 50 deletions(-) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 74a0f85932..7063ece8cd 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -37,6 +37,11 @@ typedef struct SMetaCfg { uint64_t lruSize; } SMetaCfg; +typedef struct { + int32_t nCols; + SSchema *pSchema; +} SSchemaWrapper; + typedef SVCreateTbReq STbCfg; // SMeta operations @@ -48,7 +53,9 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // For Query -STbCfg *metaGetTableInfo(SMeta *pMeta, char *tbname); +STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 78214ce14d..fdb9837292 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -31,10 +31,10 @@ #include "vnodeCommit.h" #include "vnodeFS.h" #include "vnodeMemAllocator.h" +#include "vnodeQuery.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" -#include "vnodeQuery.h" #ifdef __cplusplus extern "C" { @@ -62,6 +62,7 @@ typedef struct SVnodeMgr { extern SVnodeMgr vnodeMgr; struct SVnode { + int32_t vgId; char* path; SVnodeCfg config; SVState state; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index a0bdfed6fb..91f5fc8012 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -45,25 +45,70 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { -#if 0 - STableInfoMsg *pReq = (STableInfoMsg *)(pMsg->pCont); - STableMetaMsg *pRspMsg; - int ret; + STableInfoMsg * pReq = (STableInfoMsg *)(pMsg->pCont); + STbCfg * pTbCfg = NULL; + STbCfg * pStbCfg = NULL; + tb_uid_t uid; + int32_t nCols; + int32_t nTagCols; + SSchemaWrapper *pSW; + STableMetaMsg * pTbMetaMsg; + SSchema * pTagSchema; - if (metaGetTableInfo(pVnode->pMeta, pReq->tableFname, &pRspMsg) < 0) { + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); + if (pTbCfg == NULL) { + return NULL; + } + + if (pTbCfg->type == META_CHILD_TABLE) { + pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); + if (pStbCfg == NULL) { + return NULL; + } + + pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true); + } else { + pSW = metaGetTableSchema(pVnode->pMeta, uid, 0, true); + } + + nCols = pSW->nCols; + if (pTbCfg->type == META_SUPER_TABLE) { + nTagCols = pTbCfg->stbCfg.nTagCols; + pTagSchema = pTbCfg->stbCfg.pTagSchema; + } else if (pTbCfg->type == META_SUPER_TABLE) { + nTagCols = pStbCfg->stbCfg.nTagCols; + pTagSchema = pStbCfg->stbCfg.pTagSchema; + } else { + nTagCols = 0; + pTagSchema = NULL; + } + + pTbMetaMsg = (STableMetaMsg *)calloc(1, sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols)); + if (pTbMetaMsg == NULL) { return -1; } - *pRsp = malloc(sizeof(SRpcMsg)); - if (TD_IS_NULL(*pRsp)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pMsg); - return -1; + strcpy(pTbMetaMsg->tbFname, pTbCfg->name); + if (pTbCfg->type == META_CHILD_TABLE) { + strcpy(pTbMetaMsg->stbFname, pStbCfg->name); + pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); + } + pTbMetaMsg->numOfTags = htonl(nTagCols); + pTbMetaMsg->numOfColumns = htonl(nCols); + pTbMetaMsg->tableType = pTbCfg->type; + pTbMetaMsg->tuid = htobe64(uid); + pTbMetaMsg->vgId = htonl(pVnode->vgId); + + memcpy(pTbMetaMsg->pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols); + if (nTagCols) { + memcpy(POINTER_SHIFT(pTbMetaMsg->pSchema, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); } - // TODO - (*pRsp)->pCont = pRspMsg; + for (int i = 0; i < nCols + nTagCols; i++) { + SSchema *pSch = pTbMetaMsg->pSchema + i; + pSch->colId = htonl(pSch->colId); + pSch->bytes = htonl(pSch->bytes); + } -#endif return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index e70dcff7ad..d5291340e6 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -38,11 +38,6 @@ struct SMetaDB { DB_ENV *pEvn; }; -typedef struct { - int32_t nCols; - SSchema *pSchema; -} SSchemaWrapper; - typedef int (*bdbIdxCbPtr)(DB *, const DBT *, const DBT *, DBT *); static SMetaDB *metaNewDB(); @@ -60,9 +55,6 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT * static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); -static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); -static STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); -static STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -439,31 +431,7 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { } /* ------------------------ FOR QUERY ------------------------ */ -STbCfg *metaGetTableInfo(SMeta *pMeta, char *tbname) { - STbCfg * pTbCfg = NULL; - STbCfg * pStbCfg = NULL; - tb_uid_t uid; - int32_t sver = 0; - SSchemaWrapper *pSW; - - pTbCfg = metaGetTbInfoByName(pMeta, tbname, &uid); - if (pTbCfg == NULL) { - return NULL; - } - - if (pTbCfg->type == META_CHILD_TABLE) { - uid = pTbCfg->ctbCfg.suid; - } - - pSW = metaGetTableSchema(pMeta, uid, 0, false); - if (pSW == NULL) { - return NULL; - } - - return pTbCfg; -} - -static STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { +STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { STbCfg * pTbCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; @@ -491,7 +459,7 @@ static STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { return pTbCfg; } -static STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { +STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { STbCfg * pTbCfg = NULL; SMetaDB *pDB = pMeta->pDB; DBT key = {0}; @@ -521,7 +489,7 @@ static STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { return pTbCfg; } -static SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { uint32_t nCols; SSchemaWrapper *pSW = NULL; SMetaDB * pDB = pMeta->pDB; From bbf1c53b6d97f230097e266ea3b115c412dc1ca3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 07:07:16 +0000 Subject: [PATCH 37/97] more --- source/dnode/vnode/impl/src/vnodeQuery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 91f5fc8012..170319681a 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -57,13 +57,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); if (pTbCfg == NULL) { - return NULL; + return -1; } if (pTbCfg->type == META_CHILD_TABLE) { pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); if (pStbCfg == NULL) { - return NULL; + return -1; } pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true); From cf87dcbcff91c7b8e3ee530174101094b4f28854 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 29 Dec 2021 23:09:07 -0800 Subject: [PATCH 38/97] fix invalid refcount --- include/util/tlog.h | 1 - source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 5 +++-- source/dnode/mnode/impl/src/mndMnode.c | 1 + source/dnode/mnode/impl/src/mndTrans.c | 4 ++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/util/tlog.h b/include/util/tlog.h index 5c91398cdc..a367243a46 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -37,7 +37,6 @@ extern int32_t mqttDebugFlag; extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; -extern int32_t odbcDebugFlag; extern int32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t sDebugFlag; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 338024c20a..e09ba19164 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -117,7 +117,7 @@ typedef struct { int64_t rebootTime; int64_t lastAccessTime; int32_t accessTimes; - int16_t numOfVnodes; + int32_t numOfVnodes; int32_t numOfSupportVnodes; int32_t numOfCores; EDndReason offlineReason; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ca20ba61a1..1c7f8544e9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -196,6 +196,8 @@ static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { sdbCancelFetch(pSdb, pIter); return pDnode; } + + sdbRelease(pSdb, pDnode); } return NULL; @@ -419,10 +421,9 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; - + pCreate->port = htonl(pCreate->port); mDebug("dnode:%s:%d, start to create", pCreate->fqdn, pCreate->port); - pCreate->port = htonl(pCreate->port); if (pCreate->fqdn[0] == 0 || pCreate->port <= 0 || pCreate->port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index b76d72d79c..ad3d5e1cf6 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -219,6 +219,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { } pEpSet->numOfEps++; + sdbRelease(pSdb, pObj); } } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9263fca695..194ee55da9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -437,7 +437,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { - mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to encode while sync trans since %s", pTrans->id, terrstr()); return -1; } sdbSetRawStatus(pRaw, SDB_STATUS_READY); @@ -835,7 +835,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { - mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to encode while finish trans since %s", pTrans->id, terrstr()); } sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); From b91eb6b80b3c8ffbd771ea0e157c24d14661adaf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 15:19:09 +0800 Subject: [PATCH 39/97] [td-11818] refactor APIs, and fix bug in show tables; --- include/common/tmsg.h | 6 +- include/libs/catalog/catalog.h | 4 +- include/libs/parser/parsenodes.h | 1 + source/client/inc/clientInt.h | 11 +- source/client/src/clientImpl.c | 24 +- source/client/src/clientMsgHandler.c | 39 +- source/client/test/clientTests.cpp | 581 +++++++++++----------- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/dnode/mnode/impl/src/mndShow.c | 48 +- source/libs/catalog/src/catalog.c | 13 +- source/libs/catalog/test/catalogTests.cpp | 2 +- source/libs/parser/inc/parserInt.h | 2 +- source/libs/parser/src/astToMsg.c | 23 +- source/libs/parser/src/dCDAstProcess.c | 28 +- source/libs/parser/src/parser.c | 8 +- source/libs/parser/test/parserTests.cpp | 26 +- source/libs/parser/test/plannerTest.cpp | 4 +- source/libs/parser/test/tokenizerTest.cpp | 2 +- source/libs/planner/test/plannerTests.cpp | 2 +- source/libs/qworker/src/qworker.c | 33 +- 21 files changed, 440 insertions(+), 421 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 93d83eb419..0df910b399 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -503,7 +503,7 @@ typedef struct { typedef struct { SMsgHead header; union { - int32_t showId; + int64_t showId; int64_t qhandle; int64_t qId; }; // query handle @@ -823,7 +823,7 @@ typedef struct { } SCompactMsg; typedef struct SShowRsp { - int32_t showId; + int64_t showId; STableMetaMsg tableMeta; } SShowRsp; @@ -1330,7 +1330,7 @@ typedef struct SVShowTablesReq { } SVShowTablesReq; typedef struct SVShowTablesRsp { - int32_t id; + int64_t id; STableMetaMsg metaInfo; } SVShowTablesRsp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 8f4b9e1807..1bd29ce396 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -54,11 +54,11 @@ int32_t catalogInit(SCatalogCfg *cfg); /** * Get a cluster's catalog handle for all later operations. - * @param clusterId (input, end with \0) + * @param clusterId * @param catalogHandle (output, NO need to free it) * @return error code */ -int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 980219a4e9..041adbb582 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -169,6 +169,7 @@ typedef struct SDclStmtInfo { SEpSet epSet; char* pMsg; int32_t msgLen; + void* pExtension; // todo remove it soon } SDclStmtInfo; #ifdef __cplusplus diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 021f650254..165ea9b62f 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,10 +101,17 @@ typedef struct SReqResultInfo { uint32_t current; } SReqResultInfo; +typedef struct SShowReqInfo { + int64_t execId; // showId/queryId + int32_t vgId; + SArray *pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. +} SShowReqInfo; + typedef struct SRequestSendRecvBody { tsem_t rspSem; // not used now void* fp; - int64_t execId; // showId/queryId + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; SReqResultInfo resInfo; } SRequestSendRecvBody; @@ -131,7 +138,7 @@ extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); -SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj*); +SMsgSendInfo* buildMsgInfoImpl(SRequestObj*); int taos_init(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index efe327242a..62ef63c858 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -156,19 +156,12 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { }; cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - // todo OPT performance - char buf[12] = {0}; - sprintf(buf, "%"PRId64, pTscObj->pAppInfo->clusterId); - - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(buf, &pCatalog); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.ctx.pCatalog); if (code != TSDB_CODE_SUCCESS) { tfree(cxt.ctx.db); return code; } - cxt.ctx.pCatalog = pCatalog; code = qParseQuerySql(&cxt, pQuery); tfree(cxt.ctx.db); @@ -181,10 +174,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen}; STscObj* pTscObj = pRequest->pTscObj; - SMsgSendInfo* pSendMsg = buildSendMsgInfoImpl(pRequest); + SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; - if (pDcl->msgType == TDMT_VND_CREATE_TABLE) { + if (pDcl->msgType == TDMT_VND_CREATE_TABLE || pDcl->msgType == TDMT_VND_SHOW_TABLES) { + if (pDcl->msgType == TDMT_VND_SHOW_TABLES) { + SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; + if (pShowReqInfo->pArray == NULL) { + pShowReqInfo->currentIndex = 0; + pShowReqInfo->pArray = pDcl->pExtension; + } + } asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } else { SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; @@ -243,7 +243,7 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; int64_t transporterId = 0; @@ -498,7 +498,7 @@ void* doFetchRow(SRequestObj* pRequest) { // do nothing } - SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 6575102f81..e402403496 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -77,7 +77,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { +SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; @@ -92,7 +92,7 @@ SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { return NULL; } - pRetrieveMsg->showId = htonl(pRequest->body.execId); + pRetrieveMsg->showId = htobe64(pRequest->body.showInfo.execId); pMsgSendInfo->msgInfo.pData = pRetrieveMsg; pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); } else { @@ -101,8 +101,9 @@ SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { return NULL; } - pFetchMsg->id = htonl(pRequest->body.execId); - pFetchMsg->head.vgId = htonl(13); + pFetchMsg->id = htobe64(pRequest->body.showInfo.execId); + pFetchMsg->head.vgId = htonl(pRequest->body.showInfo.vgId); + pMsgSendInfo->msgInfo.pData = pFetchMsg; pMsgSendInfo->msgInfo.len = sizeof(SVShowTablesFetchReq); } @@ -124,7 +125,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { } SShowRsp* pShow = (SShowRsp *)pMsg->pData; - pShow->showId = htonl(pShow->showId); + pShow->showId = htobe64(pShow->showId); STableMetaMsg *pMetaMsg = &(pShow->tableMeta); pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); @@ -154,7 +155,17 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); - pRequest->body.execId = pShow->showId; + pRequest->body.showInfo.execId = pShow->showId; + + // todo + if (pRequest->type == TDMT_VND_SHOW_TABLES) { + SShowReqInfo* pShowInfo = &pRequest->body.showInfo; + + int32_t index = pShowInfo->currentIndex; + SVgroupInfo* pInfo = taosArrayGet(pShowInfo->pArray, index); + pShowInfo->vgId = pInfo->vgId; + } + tsem_post(&pRequest->body.rspSem); return 0; } @@ -184,17 +195,17 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, - pRetrieve->completed, pRequest->body.execId); + pRetrieve->completed, pRequest->body.showInfo.execId); tsem_post(&pRequest->body.rspSem); return 0; } int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { - assert(pMsg->len >= sizeof(SRetrieveTableRsp)); - SRequestObj* pRequest = param; - tfree(pRequest->body.resInfo.pRspMsg); + + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + tfree(pResInfo->pRspMsg); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -202,14 +213,14 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - pRequest->body.resInfo.pRspMsg = pMsg->pData; + assert(pMsg->len >= sizeof(SRetrieveTableRsp)); + + pResInfo->pRspMsg = pMsg->pData; SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *) pMsg->pData; pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows); pFetchRsp->precision = htons(pFetchRsp->precision); - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pFetchRsp->numOfRows; pResInfo->pData = pFetchRsp->data; @@ -218,7 +229,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pFetchRsp->numOfRows, - pFetchRsp->completed, pRequest->body.execId); + pFetchRsp->completed, pRequest->body.showInfo.execId); tsem_post(&pRequest->body.rspSem); return 0; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index be6e048378..d6d71d5fc3 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -49,51 +49,52 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); } -#if 0 TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - taos_close(pConn); -} - -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + if (pConn == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } - - taos_free_result(pRes); taos_close(pConn); } -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} TEST(testCase, show_user_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -114,18 +115,18 @@ TEST(testCase, show_user_Test) { taos_close(pConn); } -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} TEST(testCase, show_db_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -146,260 +147,259 @@ TEST(testCase, show_db_Test) { taos_close(pConn); } -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - -TEST(testCase, drop_db_test) { +//TEST(testCase, create_db_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); // if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); // } // taos_free_result(pRes); // -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); // if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); // } // taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); // taos_close(pConn); -} - - TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - // assert(pConn != NULL); - // - // TAOS_RES* pRes = taos_query(pConn, "use abc1"); - // taos_free_result(pRes); - // - // pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - // taos_free_result(pRes); - // - // taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); // if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} + +//TEST(testCase, drop_db_test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// showDB(pConn); +//// +//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// showDB(pConn); +//// +//// pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// taos_close(pConn); +//} + +// TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_table_Test) { +// // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// // assert(pConn != NULL); +// // +// // TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// // taos_free_result(pRes); +// // +// // pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// // taos_free_result(pRes); +// // +// // taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +//// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +//// } +//// +//// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); // } // // taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} -#endif +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} //TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -450,7 +450,6 @@ TEST(testCase, show_table_Test) { } TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); int32_t numOfFields = taos_num_fields(pRes); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b93fc0951c..419af6df88 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -293,7 +293,7 @@ typedef struct { } SFuncObj; typedef struct { - int32_t id; + int64_t id; int8_t type; int8_t replica; int16_t numOfColumns; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index daa87af1f5..15ff65a8fc 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -41,7 +41,7 @@ typedef struct { } SMnodeStep; typedef struct { - int32_t showId; + int64_t showId; ShowMetaFp metaFps[TSDB_MGMT_TABLE_MAX]; ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index af633355ef..d4b4459176 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -20,7 +20,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); -static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); +static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); @@ -52,8 +52,8 @@ void mndCleanupShow(SMnode *pMnode) { static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { SShowMgmt *pMgmt = &pMnode->showMgmt; - int32_t showId = atomic_add_fetch_32(&pMgmt->showId, 1); - if (showId == 0) atomic_add_fetch_32(&pMgmt->showId, 1); + int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1); + if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1); int32_t size = sizeof(SShowObj) + pMsg->payloadLen; SShowObj showObj = {0}; @@ -65,14 +65,14 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { memcpy(showObj.payload, pMsg->payload, pMsg->payloadLen); int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; - SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), &showObj, size, keepTime); + SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to put into cache since %s", showId, terrstr()); + mError("show:0x%"PRIx64", failed to put into cache since %s", showId, terrstr()); return NULL; } - mTrace("show:%d, is created, data:%p", showId, pShow); + mTrace("show:0x%"PRIx64", is created, data:%p", showId, pShow); return pShow; } @@ -87,25 +87,25 @@ static void mndFreeShowObj(SShowObj *pShow) { } } - mTrace("show:%d, is destroyed, data:%p", pShow->id, pShow); + mTrace("show:0x%d, is destroyed, data:%p", pShow->id, pShow); } -static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) { +static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId) { SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowObj *pShow = taosCacheAcquireByKey(pMgmt->cache, &showId, sizeof(int32_t)); + SShowObj *pShow = taosCacheAcquireByKey(pMgmt->cache, &showId, sizeof(showId)); if (pShow == NULL) { - mError("show:%d, already destroyed", showId); + mError("show:0x%"PRIx64", already destroyed", showId); return NULL; } - mTrace("show:%d, acquired from cache, data:%p", pShow->id, pShow); + mTrace("show:0x%"PRIx64", acquired from cache, data:%p", pShow->id, pShow); return pShow; } static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { if (pShow == NULL) return; - mTrace("show:%d, released from cache, data:%p force:%d", pShow->id, pShow, forceRemove); + mTrace("show:0x%"PRIx64", released from cache, data:%p force:%d", pShow->id, pShow, forceRemove); // A bug in tcache.c forceRemove = 0; @@ -146,18 +146,18 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { if (pRsp == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type)); + mError("show:0x%"PRIx64", failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type)); return -1; } int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta); - mDebug("show:%d, get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow->numOfRows, + mDebug("show:0x%"PRIx64", get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow->numOfRows, pShow->numOfColumns, mndShowStr(type), tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; - pMnodeMsg->pCont = pRsp; - pRsp->showId = htonl(pShow->id); + pMnodeMsg->pCont = pRsp; + pRsp->showId = htobe64(pShow->id); mndReleaseShowObj(pShow, false); return TSDB_CODE_SUCCESS; } else { @@ -175,7 +175,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { int32_t rowsRead = 0; SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont; - int32_t showId = htonl(pRetrieve->showId); + int64_t showId = htobe64(pRetrieve->showId); SShowObj *pShow = mndAcquireShowObj(pMnode, showId); if (pShow == NULL) { @@ -188,15 +188,15 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); terrno = TSDB_CODE_MSG_NOT_PROCESSED; - mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:0x%"PRIx64", failed to retrieve data since %s", pShow->id, terrstr()); return -1; } - mDebug("show:%d, start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads, + mDebug("show:0x%"PRIx64", start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads, pShow->numOfRows, mndShowStr(pShow->type)); if (mndCheckRetrieveFinished(pShow)) { - mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); + mDebug("show:0x%"PRIx64", read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); pShow->numOfReads = pShow->numOfRows; } @@ -219,7 +219,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (pRsp == NULL) { mndReleaseShowObj(pShow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:0x%"PRIx64", failed to retrieve data since %s", pShow->id, terrstr()); return -1; } @@ -228,7 +228,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); } - mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); + mDebug("show:0x%"PRIx64", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision @@ -238,10 +238,10 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; - mDebug("show:%d, retrieve completed", pShow->id); + mDebug("show:0x%"PRIx64", retrieve completed", pShow->id); mndReleaseShowObj(pShow, true); } else { - mDebug("show:%d, retrieve not completed yet", pShow->id); + mDebug("show:0x%"PRIx64", retrieve not completed yet", pShow->id); mndReleaseShowObj(pShow, false); } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b632ac772c..43f613f654 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -465,7 +465,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; } - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (NULL == ctgMgmt.pCluster) { CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); } @@ -473,8 +473,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle) { - if (NULL == clusterId || NULL == catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { + if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -483,8 +483,7 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } - size_t clen = strlen(clusterId); - SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, clusterId, clen); + SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; @@ -497,8 +496,8 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { - ctgError("put cluster %s cache to hash failed", clusterId); + if (taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES)) { + ctgError("put cluster %"PRIx64" cache to hash failed", clusterId); tfree(clusterCtg); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 0ad00046cd..aa35e56552 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -47,7 +47,7 @@ int32_t ctgTestTagNum = 1; int32_t ctgTestSVersion = 1; int32_t ctgTestTVersion = 1; -char *ctgTestClusterId = "cluster1"; +uint64_t ctgTestClusterId = 0x1; char *ctgTestDbname = "1.db1"; char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 186a4869e6..4bbe6ab907 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -102,7 +102,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* p * Destroy the meta data request structure. * @param pMetaInfo */ -void qParserClearupMetaRequestInfo(SCatalogReq* pMetaInfo); +void qParserCleanupMetaRequestInfo(SCatalogReq* pMetaInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 6b0e0ed828..792c0db266 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -26,15 +26,13 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in } SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { - SCreateAcctMsg* pMsg = (SCreateAcctMsg*)calloc(1, sizeof(SCreateAcctMsg)); - if (pMsg == NULL) { - // tscError("0x%" PRIx64 " failed to malloc for query msg", id); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + SCreateAcctMsg *pCreateMsg = (SCreateAcctMsg *) calloc(1, sizeof(SCreateAcctMsg)); + if (pCreateMsg == NULL) { + qError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } - SCreateAcctMsg *pCreateMsg = (SCreateAcctMsg *) calloc(1, sizeof(SCreateAcctMsg)); - SToken *pName = &pInfo->pMiscInfo->user.user; SToken *pPwd = &pInfo->pMiscInfo->user.passwd; @@ -67,17 +65,18 @@ SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in } *outputLen = sizeof(SCreateAcctMsg); - return pMsg; + return pCreateMsg; } + SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); if (pName->n >= TSDB_USER_LEN) { return NULL; } - SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg)); if (pMsg == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -338,7 +337,7 @@ SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* p SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid host name (name too long, maximum length 128)"; const char* msg2 = "dnode name can not be string"; - const char* msg3 = "port should be an integer that is less than 65535"; + const char* msg3 = "port should be an integer that is less than 65535 and greater than 0"; const char* msg4 = "failed prepare create dnode message"; if (taosArrayGetSize(pInfo->pMiscInfo->a) != 2) { @@ -362,7 +361,7 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs int64_t val = 0; toInteger(port->z, port->n, 10, &val, &isSign); - if (val >= UINT16_MAX) { + if (val >= UINT16_MAX || val <= 0) { buildInvalidOperationMsg(pMsgBuf, msg3); return NULL; } @@ -383,7 +382,6 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); - char* end = NULL; SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg)); pDrop->dnodeId = strtoll(pzName->z, &end, 10); @@ -397,5 +395,4 @@ SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf } return pDrop; -} - +} \ No newline at end of file diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 8a09ea9ed8..6aa4ad8275 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { } static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, - SEpSet* pEpSet, SMsgBuf* pMsgBuf) { + SEpSet* pEpSet, void** pExtension, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid name"; const char* msg2 = "wildcard string should be less than %d characters"; const char* msg3 = "database name too long"; @@ -33,12 +33,30 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE) { SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); - *pEpSet = pCtx->mgmtEpSet; - // catalogGetDBVgroupVersion() - pShowReq->head.vgId = htonl(13); + SArray* array = NULL; + SName name = {0}; + tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); + + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + + catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, 0, &array); + + SVgroupInfo* info = taosArrayGet(array, 0); + pShowReq->head.vgId = htonl(info->vgId); + pEpSet->numOfEps = info->numOfEps; + pEpSet->inUse = info->inUse; + + for(int32_t i = 0; i < pEpSet->numOfEps; ++i) { + strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + pEpSet->port[i] = info->epAddr[i].port; + } + *outputLen = sizeof(SVShowTablesReq); *output = pShowReq; + + *pExtension = array; } else { if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { SToken* pDbPrefixToken = &pShowInfo->prefix; @@ -621,7 +639,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_SHOW: { SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; - code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, pMsgBuf); + code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, &pDcl->pExtension, pMsgBuf); pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE)? TDMT_VND_SHOW_TABLES:TDMT_MND_SHOW; break; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 2ccd76723b..710cf4b5d0 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -58,13 +58,11 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { } else { SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code. return terrno; } - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(NULL, &pCatalog); - code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, pCxt->ctx.requestId, pCxt->pMsg, pCxt->msgLen); + int32_t code = qParserValidateSqlNode(pCxt->ctx.pCatalog, &info, pQueryInfo, pCxt->ctx.requestId, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } @@ -220,7 +218,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* p return code; } -void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { +void qParserCleanupMetaRequestInfo(SCatalogReq* pMetaReq) { if (pMetaReq == NULL) { return; } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 423aa1a7be..a67a9a8be8 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -99,7 +99,7 @@ void sqlCheck(const char* sql, bool valid) { } destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -159,7 +159,7 @@ TEST(testCase, validateAST_test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -205,7 +205,7 @@ TEST(testCase, function_Test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -251,7 +251,7 @@ TEST(testCase, function_Test2) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -296,7 +296,7 @@ TEST(testCase, function_Test3) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 6); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -342,7 +342,7 @@ TEST(testCase, function_Test4) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -393,7 +393,7 @@ TEST(testCase, function_Test5) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -507,7 +507,7 @@ TEST(testCase, function_Test6) { ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -569,7 +569,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, numOfCols); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -625,7 +625,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -648,7 +648,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(ret, 0); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -678,7 +678,7 @@ TEST(testCase, function_Test6) { ASSERT_NE(ret, 0); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); //=============================================================================================================== info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); @@ -700,7 +700,7 @@ TEST(testCase, function_Test6) { ASSERT_NE(ret, 0); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index f9b1722106..04c3a7d81a 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -102,7 +102,7 @@ void generateLogicplan(const char* sql) { printf("%s\n", str); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } } @@ -163,7 +163,7 @@ TEST(testCase, planner_test) { printf("%s\n", str); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 23c0aae15f..3ab6a6531c 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -714,7 +714,7 @@ TEST(testCase, extractMeta_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 11a31d15eb..4b408e67db 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -100,6 +100,6 @@ TEST(testCase, planner_test) { // ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); // // destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); +// qParserCleanupMetaRequestInfo(&req); // destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index d5ecd40ccd..808c1e19f9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -137,7 +137,6 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, return TSDB_CODE_SUCCESS; } - static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); @@ -156,8 +155,6 @@ static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t s return TSDB_CODE_SUCCESS; } - - static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } @@ -181,12 +178,10 @@ static int32_t qwAcquireTask(int32_t rwType, SQWSchStatus *sch, uint64_t qId, ui return qwAcquireTaskImpl(rwType, sch, qId, tId, task); } - static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { QW_UNLOCK(rwType, &sch->tasksLock); } - int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int8_t status, int32_t eOpt, SQWTaskStatus **task) { int32_t code = 0; @@ -233,7 +228,6 @@ int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t return TSDB_CODE_SUCCESS; } - static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) { SQWSchStatus *tsch = NULL; QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &tsch, QW_NOT_EXIST_ADD)); @@ -252,8 +246,6 @@ static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_RET(code); } - - static FORCE_INLINE int32_t qwAcquireTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerResCache **res) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); @@ -445,8 +437,6 @@ _return: QW_RET(code); } - - int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -480,7 +470,6 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t return TSDB_CODE_SUCCESS; } - int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -548,8 +537,6 @@ _return: QW_RET(code); } - - int32_t qwBuildAndSendQueryRsp(SRpcMsg *pMsg, int32_t code) { SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -648,7 +635,6 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { }; rpcSendResponse(&rpcRsp); - return TSDB_CODE_SUCCESS; } @@ -670,40 +656,43 @@ int32_t qwBuildAndSendDropRsp(SRpcMsg *pMsg, int32_t code) { int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { int32_t numOfCols = 6; - SVShowTablesRsp *pRsp = (SVShowTablesRsp *)rpcMallocCont(sizeof(SVShowTablesRsp) + sizeof(SSchema) * numOfCols); + int32_t msgSize = sizeof(SVShowTablesRsp) + sizeof(SSchema) * numOfCols; + + SVShowTablesRsp *pRsp = (SVShowTablesRsp *)rpcMallocCont(msgSize); int32_t cols = 0; SSchema *pSchema = pRsp->metaInfo.pSchema; const SSchema *s = tGetTbnameColumnSchema(); - *pSchema = createSchema(s->type, htonl(s->bytes), htonl(cols++), "name"); + *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "name"); pSchema++; int32_t type = TSDB_DATA_TYPE_TIMESTAMP; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "created"); + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "created"); pSchema++; type = TSDB_DATA_TYPE_SMALLINT; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "columns"); + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "columns"); pSchema++; - *pSchema = createSchema(s->type, htonl(s->bytes), htonl(cols++), "stable"); + *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "stable"); pSchema++; type = TSDB_DATA_TYPE_BIGINT; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "uid"); + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "uid"); pSchema++; type = TSDB_DATA_TYPE_INT; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(cols++), "vgId"); + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "vgId"); + assert(cols == numOfCols); pRsp->metaInfo.numOfColumns = htonl(cols); SRpcMsg rpcMsg = { .handle = pMsg->handle, .ahandle = pMsg->ahandle, .pCont = pRsp, - .contLen = sizeof(*pRsp), + .contLen = msgSize, .code = code, }; From 2c5311abbd65290b2ef3b15a65fc3ffa5808f8f9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 07:40:04 +0000 Subject: [PATCH 40/97] more --- include/dnode/vnode/meta/meta.h | 6 ++++ source/dnode/vnode/meta/src/metaBDBImpl.c | 40 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 7063ece8cd..b75e478add 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -42,6 +42,8 @@ typedef struct { SSchema *pSchema; } SSchemaWrapper; +typedef struct SMTbCursor SMTbCursor; + typedef SVCreateTbReq STbCfg; // SMeta operations @@ -57,6 +59,10 @@ STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); +SMTbCursor * metaOpenTbCursor(SMeta *pMeta); +void metaCloseTbCursor(SMTbCursor *pTbCur); +char *metaTbCursorNext(SMTbCursor *pTbCur); + // Options void metaOptionsInit(SMetaCfg *pMetaCfg); void metaOptionsClear(SMetaCfg *pMetaCfg); diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index d5291340e6..1f693346cc 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -541,4 +541,44 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo } return pSW; +} + +struct SMTbCursor { + DBC *pCur; +}; + +SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { + SMTbCursor *pTbCur = NULL; + SMetaDB * pDB = pMeta->pDB; + + pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); + if (pTbCur == NULL) { + return NULL; + } + + pDB->pTbDB->cursor(pDB->pTbDB, NULL, &(pTbCur->pCur), 0); + + return pTbCur; +} + +void metaCloseTbCursor(SMTbCursor *pTbCur) { + if (pTbCur) { + if (pTbCur->pCur) { + pTbCur->pCur->close(pTbCur->pCur); + } + free(pTbCur); + } +} + +char *metaTbCursorNext(SMTbCursor *pTbCur) { + DBT key = {0}; + DBT value = {0}; + STbCfg tbCfg; + + if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { + metaDecodeTbInfo(&(value.data), &tbCfg); + return tbCfg.name; + } else { + return NULL; + } } \ No newline at end of file From ad44f5e01115ab80061af886af4a4944a81a40a6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Dec 2021 00:01:27 -0800 Subject: [PATCH 41/97] Invalid raw data len while create vgroup --- source/dnode/mnode/impl/src/mndTrans.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 194ee55da9..cf434dda39 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -79,17 +79,17 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { for (int32_t i = 0; i < redoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); - rawDataLen += sdbGetRawTotalSize(pTmp); + rawDataLen += (sdbGetRawTotalSize(pTmp) + 4); } for (int32_t i = 0; i < undoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); - rawDataLen += sdbGetRawTotalSize(pTmp); + rawDataLen += (sdbGetRawTotalSize(pTmp) + 4); } for (int32_t i = 0; i < commitLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); - rawDataLen += sdbGetRawTotalSize(pTmp); + rawDataLen += (sdbGetRawTotalSize(pTmp) + 4); } for (int32_t i = 0; i < redoActionNum; ++i) { From 236b8bc3c8db11baa7b474b56ac194140b739bc5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 30 Dec 2021 17:03:00 +0800 Subject: [PATCH 42/97] handle read/write crash concurrently --- source/libs/index/inc/index_fst.h | 13 ++-- source/libs/index/inc/index_tfile.h | 1 + source/libs/index/src/index.c | 4 +- source/libs/index/src/index_cache.c | 13 ++-- source/libs/index/src/index_fst.c | 80 +++++++++++--------- source/libs/index/src/index_fst_automation.c | 24 ++---- source/libs/index/src/index_tfile.c | 25 +++--- source/libs/index/test/fstTest.cc | 45 ++++++++++- source/libs/index/test/indexTests.cc | 34 +++++++-- 9 files changed, 152 insertions(+), 87 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 73c79b2619..072ec93e4e 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -142,7 +142,8 @@ uint64_t fstStateInputLen(FstState* state); // end_addr uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data); uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes); -uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, + uint64_t nTrans); // input uint8_t fstStateInput(FstState* state, FstNode* node); uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i); @@ -255,9 +256,10 @@ typedef struct FstMeta { } FstMeta; typedef struct Fst { - FstMeta* meta; - FstSlice* data; // - FstNode* root; // + FstMeta* meta; + FstSlice* data; // + FstNode* root; // + pthread_mutex_t mtx; } Fst; // refactor simple function @@ -310,7 +312,8 @@ StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* sta void swsResultDestroy(StreamWithStateResult* result); typedef void* (*StreamCallback)(void*); -StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max); +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, + FstBoundWithData* max); void streamWithStateDestroy(StreamWithState* sws); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index d8e69b5fde..4928e01a63 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -77,6 +77,7 @@ typedef struct TFileReader { Fst* fst; WriterCtx* ctx; TFileHeader header; + bool remove; } TFileReader; typedef struct IndexTFile { diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 6d64bdbd65..167d33f51f 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -94,7 +94,6 @@ void indexClose(SIndex* sIdx) { #endif #ifdef USE_INVERTED_INDEX - indexCacheDestroy(sIdx->cache); void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; @@ -104,6 +103,7 @@ void indexClose(SIndex* sIdx) { taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); #endif + free(sIdx->path); free(sIdx); return; } @@ -459,7 +459,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { } else { if (value->val != NULL) { taosArrayClear(value->val); } } - // free(value->colVal); + free(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 882091afb9..bce9e5fb87 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 200 +#define MEM_TERM_LIMIT 10000 * 10 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -110,7 +110,10 @@ void indexCacheDestroySkiplist(SSkipList* slt) { while (tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - if (ct != NULL) {} + if (ct != NULL) { + free(ct->colVal); + free(ct); + } } tSkipListDestroyIter(iter); tSkipListDestroy(slt); @@ -178,9 +181,9 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - pthread_mutex_unlock(&cache->mtx); + // pthread_mutex_unlock(&cache->mtx); taosMsleep(50); - pthread_mutex_lock(&cache->mtx); + // pthread_mutex_lock(&cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; @@ -271,7 +274,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV SIndexTerm* term = query->term; EIndexQueryType qtype = query->qType; CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; - indexCacheDebug(pCache); + // indexCacheDebug(pCache); int ret = indexQueryMem(mem, &ct, qtype, result, s); if (ret == 0 && *s != kTypeDeletion) { diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 18024fa391..246f50a12b 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -354,7 +354,8 @@ uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } -uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) { +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, + uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size @@ -403,8 +404,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans - - (i * tSizes) - tSizes; + uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - + node->nTrans - (i * tSizes) - tSizes; uint8_t* data = fstSliceData(slice, NULL); return unpackDelta(data + at, tSizes, node->end); } @@ -595,7 +596,8 @@ FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->nTrans = nTrans; n->sizes = sz; - n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); + n->finalOutput = + fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); } return n; } @@ -875,9 +877,7 @@ void* fstBuilderInsertInner(FstBuilder* b) { // b->wrt = NULL; return b->wrt; } -void fstBuilderFinish(FstBuilder* b) { - fstBuilderInsertInner(b); -} +void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } FstSlice fstNodeAsSlice(FstNode* node) { FstSlice* slice = &node->data; @@ -894,9 +894,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { return trn; } -void fstLastTransitionDestroy(FstLastTransition* trn) { - free(trn); -} +void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; if (trn == NULL) { return; } @@ -959,9 +957,10 @@ Fst* fstCreate(FstSlice* slice) { fst->meta->checkSum = checkSum; FstSlice* s = calloc(1, sizeof(FstSlice)); - *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; + pthread_mutex_init(&fst->mtx, NULL); return fst; FST_CREAT_FAILED: @@ -973,14 +972,18 @@ void fstDestroy(Fst* fst) { free(fst->meta); fstSliceDestroy(fst->data); free(fst->data); + pthread_mutex_destroy(&fst->mtx); } free(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { + // dec lock range + pthread_mutex_lock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; + uint8_t* data = fstSliceData(b, &len); SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); @@ -988,7 +991,10 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; - if (false == fstNodeFindInput(root, inp, &res)) { return false; } + if (false == fstNodeFindInput(root, inp, &res)) { + pthread_mutex_unlock(&fst->mtx); + return false; + } FstTransition trn; fstNodeGetTransitionAt(root, res, &trn); @@ -997,6 +1003,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { + pthread_mutex_unlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1007,13 +1014,13 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { fstNodeDestroy(*node); } taosArrayDestroy(nodes); - fst->root = NULL; + pthread_mutex_unlock(&fst->mtx); *out = tOut; - return true; } FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { + // refactor later return fstStreamBuilderCreate(fst, ctx); } StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { @@ -1021,24 +1028,28 @@ StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { + // refactor later return fstStreamBuilderCreate(fst, ctx); } FstNode* fstGetRoot(Fst* fst) { - if (fst->root != NULL) { return fst->root; } + // pthread_mutex_lock(&fst->mtx); + if (fst->root != NULL) { + // pthread_mutex_unlock(&fst->mtx); + return fst->root; + } CompiledAddr rAddr = fstGetRootAddr(fst); fst->root = fstGetNode(fst, rAddr); + // pthread_mutex_unlock(&fst->mtx); return fst->root; } + FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { + // refactor later return fstNodeCreate(fst->meta->version, addr, fst->data); } -FstType fstGetType(Fst* fst) { - return fst->meta->ty; -} -CompiledAddr fstGetRootAddr(Fst* fst) { - return fst->meta->rootAddr; -} +FstType fstGetType(Fst* fst) { return fst->meta->ty; } +CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; } Output fstEmptyFinalOutput(Fst* fst, bool* null) { Output res = 0; @@ -1053,8 +1064,7 @@ Output fstEmptyFinalOutput(Fst* fst, bool* null) { } bool fstVerify(Fst* fst) { - uint32_t checkSum = fst->meta->checkSum; - int32_t len; + uint32_t len, checkSum = fst->meta->checkSum; uint8_t* data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; if (!taosCheckChecksumWhole(data, len)) { return false; } @@ -1094,15 +1104,12 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { } } -bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { - return bound->type == Excluded ? false : true; -} +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -void fstBoundDestroy(FstBoundWithData* bound) { - free(bound); -} +void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } -StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, + FstBoundWithData* max) { StreamWithState* sws = calloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1131,7 +1138,9 @@ void streamWithStateDestroy(StreamWithState* sws) { bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { AutomationCtx* aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { - if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); } + if (fstBoundWithDataIsIncluded(min)) { + sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); + } StreamState s = {.node = fstGetRoot(sws->fst), .trans = 0, .out = {.null = false, .out = 0}, @@ -1203,7 +1212,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { uint64_t trans = s->trans; FstTransition trn; fstNodeGetTransitionAt(n, trans - 1, &trn); - StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; + StreamState s = { + .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); return true; } @@ -1260,9 +1270,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb size_t isz = taosArrayGetSize(sws->inp); uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); - for (uint32_t i = 0; i < isz; i++) { - buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); - } + for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); @@ -1327,8 +1335,8 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { } void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); - tfree(b->min); fstSliceDestroy(&b->max->data); + tfree(b->min); tfree(b->max); free(b); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 590ff294bf..c6e3cee3e3 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -17,9 +17,7 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { - return NULL; - } + if (nsv == NULL) { return NULL; } nsv->kind = kind; nsv->type = ty; @@ -37,9 +35,7 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; - if (sv == NULL) { - return; - } + if (sv == NULL) { return; } if (sv->type == FST_INT) { // @@ -52,9 +48,7 @@ void startWithStateValueDestroy(void* val) { } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { - return NULL; - } + if (nsv == NULL) { return NULL; } nsv->kind = sv->kind; nsv->type = sv->type; @@ -94,14 +88,10 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { StartWithStateValue* ssv = (StartWithStateValue*)state; - if (ssv == NULL || ctx == NULL) { - return NULL; - } + if (ssv == NULL || ctx == NULL) { return NULL; } char* data = ctx->data; - if (ssv->kind == Done) { - return startWithStateValueCreate(Done, FST_INT, &ssv->val); - } + if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; @@ -138,9 +128,7 @@ AutomationFunc automFuncs[] = { AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { - return NULL; - } + if (ctx == NULL) { return NULL; } StartWithStateValue* sv = NULL; if (atype == AUTOMATION_ALWAYS) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 17322e301e..177132c67d 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -53,13 +53,6 @@ static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); -static TFileReader* tfileReaderCreateImpl(WriterCtx* ctx) { - TFileReader* reader = tfileReaderCreate(ctx); - tfileReaderRef(reader); - // tfileSerialCacheKey(&key, buf); - return reader; -} - TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); if (tcache == NULL) { return NULL; } @@ -88,13 +81,16 @@ TFileCache* tfileCacheCreate(const char* path) { } char buf[128] = {0}; - TFileReader* reader = tfileReaderCreateImpl(wc); + TFileReader* reader = tfileReaderCreate(wc); TFileHeader* header = &reader->header; TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; tfileSerialCacheKey(&key, buf); + + tfileReaderRef(reader); + // indexTable taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } taosArrayDestroyEx(files, tfileDestroyFileName); @@ -139,6 +135,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) if (p != NULL) { TFileReader* oldReader = *p; taosHashRemove(tcache->tableCache, buf, strlen(buf)); + oldReader->remove = true; tfileReaderUnRef(oldReader); } @@ -172,7 +169,7 @@ void tfileReaderDestroy(TFileReader* reader) { if (reader == NULL) { return; } // T_REF_INC(reader); fstDestroy(reader->fst); - writerCtxDestroy(reader->ctx, true); + writerCtxDestroy(reader->ctx, reader->remove); free(reader); } @@ -232,7 +229,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); if (wc == NULL) { return NULL; } - TFileReader* reader = tfileReaderCreateImpl(wc); + TFileReader* reader = tfileReaderCreate(wc); return reader; // tfileSerialCacheKey(&key, buf); @@ -330,13 +327,16 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { return -1; } // write fst + indexError("--------Begin----------------"); for (size_t i = 0; i < sz; i++) { // TODO, fst batch write later TFileValue* v = taosArrayGetP((SArray*)data, i); if (tfileWriteData(tw, v) == 0) { // } + indexError("data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); } + indexError("--------End----------------"); fstBuilderFinish(tw->fb); fstBuilderDestroy(tw->fb); tw->fb = NULL; @@ -360,7 +360,10 @@ IndexTFile* indexTFileCreate(const char* path) { tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile* tfile) { free(tfile); } +void IndexTFileDestroy(IndexTFile* tfile) { + tfileCacheDestroy(tfile->cache); + free(tfile); +} int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index a4d8bb36f1..3b5410547d 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include "index.h" #include "indexInt.h" @@ -42,7 +43,8 @@ class FstWriter { class FstReadMemory { public: - FstReadMemory(size_t size) { + FstReadMemory(size_t size, const std::string& fileName = fileName) { + tfInit(); _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; @@ -101,6 +103,7 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); + tfCleanup(); } private: @@ -165,8 +168,44 @@ void checkFstCheckIterator() { delete m; tfCleanup(); } -int main() { - checkFstCheckIterator(); + +void fst_get(Fst* fst) { + for (int i = 0; i < 10000; i++) { + std::string term = "Hello"; + FstSlice key = fstSliceCreate((uint8_t*)term.c_str(), term.size()); + uint64_t offset = 0; + bool ret = fstGet(fst, &key, &offset); + if (ret == false) { + std::cout << "not found" << std::endl; + } else { + std::cout << "found value:" << offset << std::endl; + } + } +} + +#define NUM_OF_THREAD 10 +void validateTFile(char* arg) { + tfInit(); + + std::thread threads[NUM_OF_THREAD]; + // std::vector threads; + TFileReader* reader = tfileReaderOpen(arg, 0, 8417, "tag1"); + + for (int i = 0; i < NUM_OF_THREAD; i++) { + threads[i] = std::thread(fst_get, reader->fst); + // threads.push_back(fst_get, reader->fst); + // std::thread t(fst_get, reader->fst); + } + for (int i = 0; i < NUM_OF_THREAD; i++) { + // wait join + threads[i].join(); + } + tfCleanup(); +} +int main(int argc, char* argv[]) { + if (argc > 1) { validateTFile(argv[1]); } + // checkFstCheckIterator(); // checkFstPrefixSearch(); + return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 588205c67f..cffb09ee4d 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -332,6 +332,8 @@ class TFileObj { TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) { colId_ = 10; + reader_ = NULL; + writer_ = NULL; // Do Nothing // } @@ -527,6 +529,7 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + // indexTermDestry(term); } { std::string colVal("v3"); @@ -687,7 +690,7 @@ class IndexEnv2 : public ::testing::Test { IndexObj* index; }; TEST_F(IndexEnv2, testIndexOpen) { - std::string path = "/tmp"; + std::string path = "/tmp/test"; if (index->Init(path) != 0) { std::cout << "failed to init index" << std::endl; exit(1); @@ -723,10 +726,24 @@ TEST_F(IndexEnv2, testIndexOpen) { } indexMultiTermDestroy(terms); } - { + size_t size = 200; std::string colName("tag1"), colVal("Hello"); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = size * 3; i < size * 4; i++) { + int tableId = i; + int ret = index->Put(terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + + { + std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -735,16 +752,16 @@ TEST_F(IndexEnv2, testIndexOpen) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); index->Search(mq, result); std::cout << "target size: " << taosArrayGetSize(result) << std::endl; - // assert(taosArrayGetSize(result) == targetSize); + assert(taosArrayGetSize(result) == 400); } } TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp"; + std::string path = "/tmp/test"; if (index->Init(path) != 0) {} int numOfTable = 100 * 10000; - index->WriteMillonData("tag1", "Hello world", numOfTable); - int target = index->SearchOne("tag1", "Hello world"); + index->WriteMillonData("tag1", "Hello", numOfTable); + int target = index->SearchOne("tag1", "Hello"); assert(numOfTable == target); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { @@ -769,4 +786,7 @@ TEST_F(IndexEnv2, testIndex_performance) { std::string path = "/tmp"; if (index->Init(path) != 0) {} } -TEST_F(IndexEnv2, testIndexMultiTag) {} +TEST_F(IndexEnv2, testIndexMultiTag) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} +} From 0e8676abc89e94f46513188e415cffcddb1a7016 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 17:13:02 +0800 Subject: [PATCH 43/97] [td-11818] show tables; --- source/client/src/clientImpl.c | 27 ++++++++- source/client/test/clientTests.cpp | 72 ++++++++++++------------ source/dnode/vnode/impl/src/vnodeQuery.c | 55 +++++++++++++++++- 3 files changed, 115 insertions(+), 39 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index af28845673..050e763919 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -502,8 +502,31 @@ void* doFetchRow(SRequestObj* pRequest) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; - } else { - // do nothing + } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { + pRequest->type = TDMT_VND_SHOW_TABLES; + SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; + pShowReqInfo->currentIndex += 1; + if (pShowReqInfo->currentIndex >= taosArrayGetSize(pShowReqInfo->pArray)) { + return NULL; + } + + SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); + SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + pShowReq->head.vgId = htonl(pVgroupInfo->vgId); + + pRequest->body.requestMsg.len = sizeof(SVShowTablesReq); + pRequest->body.requestMsg.pData = pShowReq; + + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + + int64_t transporterId = 0; + STscObj *pTscObj = pRequest->pTscObj; + asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + + tsem_wait(&pRequest->body.rspSem); + destroySendMsgInfo(body); + + pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d6d71d5fc3..013cf794e3 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -96,24 +96,24 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} //TEST(testCase, drop_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -128,24 +128,24 @@ TEST(testCase, show_user_Test) { // taos_close(pConn); //} -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} //TEST(testCase, create_db_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 2dde9e03e8..a80828b6f7 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -16,6 +16,8 @@ #include "vnodeQuery.h" #include "vnodeDef.h" +static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); + int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { @@ -39,7 +41,8 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TDMT_VND_SHOW_TABLES: return qWorkerProcessShowMsg(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_SHOW_TABLES_FETCH: - return qWorkerProcessShowFetchMsg(pVnode, pVnode->pQuery, pMsg); + return vnodeGetTableList(pVnode, pMsg); +// return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -112,5 +115,55 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pSch->bytes = htonl(pSch->bytes); } + return 0; +} + +/** + * @param pVnode + * @param pMsg + * @param pRsp + */ +static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { + SMTbCursor* pCur = metaOpenTbCursor(pVnode->pMeta); + SArray* pArray = taosArrayInit(10, POINTER_BYTES); + + char* name = NULL; + int32_t totalLen = 0; + while ((name = metaTbCursorNext(pCur)) != NULL) { + taosArrayPush(pArray, &name); + totalLen += strlen(name); + } + + metaCloseTbCursor(pCur); + + int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; + int32_t numOfTables = (int32_t) taosArrayGetSize(pArray); + + int32_t payloadLen = rowLen * numOfTables; +// SVShowTablesFetchReq *pFetchReq = pMsg->pCont; + + SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp) + payloadLen); + memset(pFetchRsp, 0, sizeof(struct SVShowTablesFetchRsp) + payloadLen); + + char* p = pFetchRsp->data; + for(int32_t i = 0; i < numOfTables; ++i) { + char* n = taosArrayGetP(pArray, i); + STR_TO_VARSTR(p, n); + + p += rowLen; + } + + pFetchRsp->numOfRows = htonl(numOfTables); + pFetchRsp->precision = 0; + + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pFetchRsp, + .contLen = sizeof(SVShowTablesFetchRsp) + payloadLen, + .code = 0, + }; + + rpcSendResponse(&rpcMsg); return 0; } \ No newline at end of file From ecfd874c53fa0ea4fb3850f1d982b1f49b6c3ec5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Dec 2021 01:28:06 -0800 Subject: [PATCH 44/97] Improve SDB traversal efficiency --- include/dnode/mnode/sdb/sdb.h | 16 ++- source/dnode/mnode/impl/src/mndDb.c | 4 +- source/dnode/mnode/impl/src/mndVgroup.c | 124 +++++++++++++----------- source/dnode/mnode/sdb/src/sdbHash.c | 24 +++++ 4 files changed, 105 insertions(+), 63 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 33f9dc5a1a..3ff86bea3e 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -181,6 +181,7 @@ typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); +typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3); typedef struct { ESdbType sdbType; @@ -279,7 +280,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * * @param pSdb The sdb object. * @param type The type of the table. - * @param type The initial iterator of the table. + * @param pIter The initial iterator of the table. * @param pObj The object of the row just fetched. * @return void* The next iterator of the table. */ @@ -289,11 +290,22 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); * @brief Cancel a traversal * * @param pSdb The sdb object. - * @param pIter The iterator of the table. * @param type The initial iterator of table. */ void sdbCancelFetch(SSdb *pSdb, void *pIter); +/** + * @brief Traverse a sdb + * + * @param pSdb The sdb object. + * @param type The initial iterator of table. + * @param fp The function pointer. + * @param p1 The callback param. + * @param p2 The callback param. + * @param p3 The callback param. + */ +void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3); + /** * @brief Get the number of rows in the table * diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d05e301024..f9199f9eed 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -549,7 +549,7 @@ static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pO return 0; } -static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { SSdbRaw *pCommitRaw = mndDbActionEncode(pNewDb); if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; @@ -725,7 +725,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; - SVnodeGid * pVgid = pVgroup->vnodeGid + vn; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode == NULL) return -1; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1b0026cd13..41de70283f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -238,39 +238,48 @@ SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *p return pDrop; } +static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SDnodeObj *pDnode = pObj; + pDnode->numOfVnodes = 0; + return true; +} + +static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SDnodeObj *pDnode = pObj; + SArray *pArray = p1; + + pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); + + int64_t curMs = taosGetTimestampMs(); + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + if (online && pDnode->numOfSupportVnodes > 0) { + taosArrayPush(pArray, pDnode); + } + + bool isMnode = mndIsMnode(pMnode, pDnode->id); + + mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d", pDnode->id, pDnode->numOfVnodes, + pDnode->numOfSupportVnodes, isMnode, online); + + if (isMnode) { + pDnode->numOfVnodes++; + } + + return true; +} + static SArray *mndBuildDnodesArray(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); + SArray *pArray = taosArrayInit(numOfDnodes, sizeof(SDnodeObj)); if (pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void *pIter = NULL; - while (1) { - SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); - if (pIter == NULL) break; - - int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - - bool isMnode = mndIsMnode(pMnode, pDnode->id); - if (isMnode) { - pDnode->numOfVnodes++; - } - - int64_t curMs = taosGetTimestampMs(); - bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); - if (online) { - taosArrayPush(pArray, pDnode); - } - - mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d", pDnode->id, numOfVnodes, - pDnode->numOfSupportVnodes, isMnode, online); - sdbRelease(pSdb, pDnode); - } - + sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); + sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, NULL, NULL); return pArray; } @@ -302,7 +311,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } - mDebug("db:%s, vgId:%d, vindex:%d dnodeId:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); + mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); pDnode->numOfVnodes++; } @@ -412,6 +421,20 @@ static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SVgObj *pVgroup = pObj; + int64_t uid = *(int64_t *)p1; + int8_t *pReplica = p2; + int32_t *pNumOfVgroups = p3; + + if (pVgroup->dbUid == uid) { + *pReplica = MAX(*pReplica, pVgroup->replica); + (*pNumOfVgroups)++; + } + + return true; +} + static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); @@ -420,25 +443,10 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return -1; } - int8_t replica = 1; - int32_t numOfVgroups = 0; - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - if (pVgroup->dbUid == pDb->uid) { - replica = MAX(replica, pVgroup->replica); - numOfVgroups++; - } - - sdbRelease(pSdb, pVgroup); - } - - *pReplica = replica; - *pNumOfVgroups = numOfVgroups; + *pReplica = 1; + *pNumOfVgroups = 0; + sdbTraverse(pSdb, SDB_VGROUP, mndGetVgroupMaxReplicaFp, &pDb->uid, pReplica, pNumOfVgroups); + mndReleaseDb(pMnode, pDb); return 0; } @@ -540,25 +548,23 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { - SSdb *pSdb = pMnode->pSdb; - int32_t numOfVnodes = 0; - void *pIter = NULL; +static bool mndGetVnodesNumFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SVgObj *pVgroup = pObj; + int32_t dnodeId = *(int32_t *)p1; + int32_t *pNumOfVnodes = (int32_t *)p2; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { - numOfVnodes++; - } + for (int32_t v = 0; v < pVgroup->replica; ++v) { + if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { + (*pNumOfVnodes)++; } - - sdbRelease(pSdb, pVgroup); } + return true; +} + +int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { + int32_t numOfVnodes = 0; + sdbTraverse(pMnode->pSdb, SDB_VGROUP, mndGetVnodesNumFp, &dnodeId, &numOfVnodes, NULL); return numOfVnodes; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 06c2563910..0388fa99f5 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -336,6 +336,30 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter) { taosRUnLockLatch(pLock); } +void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3) { + SHashObj *hash = sdbGetHash(pSdb, type); + if (hash == NULL) return; + + SRWLatch *pLock = &pSdb->locks[type]; + taosRLockLatch(pLock); + + SSdbRow **ppRow = taosHashIterate(hash, NULL); + while (ppRow != NULL) { + SSdbRow *pRow = *ppRow; + if (pRow->status == SDB_STATUS_READY) { + bool isContinue = (*fp)(pSdb->pMnode, pRow->pObj, p1, p2, p3); + if (!isContinue) { + taosHashCancelIterate(hash, ppRow); + break; + } + } + + ppRow = taosHashIterate(hash, ppRow); + } + + taosRUnLockLatch(pLock); +} + int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return 0; From e1bf252db12cb6a352e55ddb36972265ccc10f56 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 09:53:29 +0000 Subject: [PATCH 45/97] fix bugs --- source/dnode/vnode/meta/src/metaBDBImpl.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 1f693346cc..aafbb1b4e7 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -378,10 +378,10 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { if (pTbCfg->type == META_SUPER_TABLE) { tsize += taosEncodeVariantU32(buf, pTbCfg->stbCfg.nTagCols); for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { - tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pSchema[i].type); - tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pSchema[i].colId); - tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pSchema[i].bytes); - tsize += taosEncodeString(buf, pTbCfg->stbCfg.pSchema[i].name); + tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pTagSchema[i].type); + tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].colId); + tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].bytes); + tsize += taosEncodeString(buf, pTbCfg->stbCfg.pTagSchema[i].name); } // tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); @@ -406,10 +406,10 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeVariantU32(buf, &(pTbCfg->stbCfg.nTagCols)); pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols); for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { - buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].bytes); - buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pSchema[i].name); + buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pTagSchema[i].type)); + buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].bytes); + buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pTagSchema[i].name); } } else if (pTbCfg->type == META_CHILD_TABLE) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); @@ -574,9 +574,11 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { DBT key = {0}; DBT value = {0}; STbCfg tbCfg; + void * pBuf; if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { - metaDecodeTbInfo(&(value.data), &tbCfg); + pBuf = value.data; + metaDecodeTbInfo(pBuf, &tbCfg); return tbCfg.name; } else { return NULL; From 82faa5cd3440e83abdbe1b73f05d343c6a97893d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 18:20:12 +0800 Subject: [PATCH 46/97] [td-11818] send table name instead of full name. --- source/dnode/mnode/impl/src/mndStb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 918f43f2bd..f81a19a780 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -207,7 +207,10 @@ static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb SMsgHead * pMsgHead; req.ver = 0; - req.name = pStb->name; + SName name = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE); + + req.name = (char*) tNameGetTableName(&name); req.ttl = 0; req.keep = 0; req.type = TD_SUPER_TABLE; From 8789ae77167f6e8651f5ab154526601ab8d77918 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 30 Dec 2021 02:33:57 -0800 Subject: [PATCH 47/97] set queue to a named type instead of void --- include/util/tqueue.h | 66 +++++------ include/util/tworker.h | 22 ++-- source/dnode/mgmt/daemon/src/daemon.c | 2 +- source/dnode/mgmt/impl/inc/dndInt.h | 4 +- source/dnode/mgmt/impl/src/dndBnode.c | 6 +- source/dnode/mgmt/impl/src/dndVnodes.c | 42 +++---- source/util/src/tqueue.c | 155 +++++++++++-------------- source/util/src/tworker.c | 12 +- 8 files changed, 141 insertions(+), 168 deletions(-) diff --git a/include/util/tqueue.h b/include/util/tqueue.h index bcb9aea856..a57bdb5ce8 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -22,59 +22,57 @@ extern "C" { /* -This set of API for queue is designed specially for vnode/mnode. The main purpose is to -consume all the items instead of one item from a queue by one single read. Also, it can -combine multiple queues into a queue set, a consumer thread can consume a queue set via +This set of API for queue is designed specially for vnode/mnode. The main purpose is to +consume all the items instead of one item from a queue by one single read. Also, it can +combine multiple queues into a queue set, a consumer thread can consume a queue set via a single API instead of looping every queue by itself. Notes: -1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe +1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe 2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe. 3: read/write operation APIs are multi-thread safe To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c) -shall be used to set up the protection. +shall be used to set up the protection. */ -typedef void *taos_queue; -typedef void *taos_qset; -typedef void *taos_qall; +typedef struct STaosQueue STaosQueue; +typedef struct STaosQset STaosQset; +typedef struct STaosQall STaosQall; typedef void (*FProcessItem)(void *ahandle, void *pItem); -typedef void (*FProcessItems)(void *ahandle, taos_qall qall, int numOfItems); +typedef void (*FProcessItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); -taos_queue taosOpenQueue(); -void taosCloseQueue(taos_queue); -void taosSetQueueFp(taos_queue, FProcessItem, FProcessItems); -void *taosAllocateQitem(int size); -void taosFreeQitem(void *pItem); -int taosWriteQitem(taos_queue, void *pItem); -int taosReadQitem(taos_queue, void **pItem); -bool taosQueueEmpty(taos_queue); +STaosQueue *taosOpenQueue(); +void taosCloseQueue(STaosQueue *queue); +void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp); +void *taosAllocateQitem(int32_t size); +void taosFreeQitem(void *pItem); +int32_t taosWriteQitem(STaosQueue *queue, void *pItem); +int32_t taosReadQitem(STaosQueue *queue, void **ppItem); +bool taosQueueEmpty(STaosQueue *queue); -taos_qall taosAllocateQall(); -void taosFreeQall(taos_qall); -int taosReadAllQitems(taos_queue, taos_qall); -int taosGetQitem(taos_qall, void **pItem); -void taosResetQitems(taos_qall); +STaosQall *taosAllocateQall(); +void taosFreeQall(STaosQall *qall); +int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall); +int32_t taosGetQitem(STaosQall *qall, void **ppItem); +void taosResetQitems(STaosQall *qall); -taos_qset taosOpenQset(); -void taosCloseQset(); -void taosQsetThreadResume(taos_qset param); -int taosAddIntoQset(taos_qset, taos_queue, void *ahandle); -void taosRemoveFromQset(taos_qset, taos_queue); -int taosGetQueueNumber(taos_qset); +STaosQset *taosOpenQset(); +void taosCloseQset(STaosQset *qset); +void taosQsetThreadResume(STaosQset *qset); +int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); +void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); +int32_t taosGetQueueNumber(STaosQset *qset); -int taosReadQitemFromQset(taos_qset, void **pItem, void **ahandle, FProcessItem *); -int taosReadAllQitemsFromQset(taos_qset, taos_qall, void **ahandle, FProcessItems *); +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp); +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp); -int taosGetQueueItemsNumber(taos_queue param); -int taosGetQsetItemsNumber(taos_qset param); +int32_t taosGetQueueItemsNumber(STaosQueue *queue); +int32_t taosGetQsetItemsNumber(STaosQset *qset); #ifdef __cplusplus } #endif #endif /*_TD_UTIL_QUEUE_H*/ - - diff --git a/include/util/tworker.h b/include/util/tworker.h index 2e5852cbba..27f03bd2b6 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -35,7 +35,7 @@ typedef struct SWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - taos_qset qset; + STaosQset *qset; const char *name; SWorker *workers; pthread_mutex_t mutex; @@ -44,8 +44,8 @@ typedef struct SWorkerPool { typedef struct SMWorker { int32_t id; // worker id pthread_t thread; // thread - taos_qall qall; - taos_qset qset; // queue set + STaosQall *qall; + STaosQset *qset; // queue set SMWorkerPool *pool; } SMWorker; @@ -57,15 +57,15 @@ typedef struct SMWorkerPool { pthread_mutex_t mutex; } SMWorkerPool; -int32_t tWorkerInit(SWorkerPool *pool); -void tWorkerCleanup(SWorkerPool *pool); -taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp); -void tWorkerFreeQueue(SWorkerPool *pool, taos_queue queue); +int32_t tWorkerInit(SWorkerPool *pool); +void tWorkerCleanup(SWorkerPool *pool); +STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp); +void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue); -int32_t tMWorkerInit(SMWorkerPool *pool); -void tMWorkerCleanup(SMWorkerPool *pool); -taos_queue tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp); -void tMWorkerFreeQueue(SMWorkerPool *pool, taos_queue queue); +int32_t tMWorkerInit(SMWorkerPool *pool); +void tMWorkerCleanup(SMWorkerPool *pool); +STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp); +void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 8161b8d125..cedab6266e 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -140,7 +140,7 @@ void dmnInitOption(SDnodeOpt *pOption) { pOption->sver = 30000000; //3.0.0.0 pOption->numOfCores = tsNumOfCores; pOption->numOfSupportVnodes = tsNumOfSupportVnodes; - pOption->numOfCommitThreads = 1; + pOption->numOfCommitThreads = tsNumOfCommitThreads; pOption->statusInterval = tsStatusInterval; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; pOption->ratioOfQueryCores = tsRatioOfQueryCores; diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 954e21aefa..07c8ce5d02 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -64,7 +64,7 @@ typedef struct { int32_t maxNum; void *queueFp; SDnode *pDnode; - taos_queue queue; + STaosQueue *queue; union { SWorkerPool pool; SMWorkerPool mpool; @@ -92,7 +92,7 @@ typedef struct { SDnodeEps *dnodeEps; pthread_t *threadId; SRWLatch latch; - taos_queue pMgmtQ; + STaosQueue *pMgmtQ; SWorkerPool mgmtPool; } SDnodeMgmt; diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 66b619318d..c12d449517 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -19,7 +19,7 @@ #include "dndTransport.h" #include "dndWorker.h" -static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); static SBnode *dndAcquireBnode(SDnode *pDnode) { SBnodeMgmt *pMgmt = &pDnode->bmgmt; @@ -286,7 +286,7 @@ static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) { taosFreeQitem(pMsg); } -static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t code) { +static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { for (int32_t i = 0; i < numOfMsgs; ++i) { SRpcMsg *pMsg = NULL; taosGetQitem(qall, (void **)&pMsg); @@ -294,7 +294,7 @@ static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t cod } } -static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { SBnode *pBnode = dndAcquireBnode(pDnode); if (pBnode == NULL) { dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 3eab3e5aec..8835e0ba65 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -27,20 +27,20 @@ typedef struct { } SWrapperCfg; typedef struct { - int32_t vgId; - int32_t refCount; - int32_t vgVersion; - int8_t dropped; - int8_t accessState; - uint64_t dbUid; - char *db; - char *path; - SVnode *pImpl; - taos_queue pWriteQ; - taos_queue pSyncQ; - taos_queue pApplyQ; - taos_queue pQueryQ; - taos_queue pFetchQ; + int32_t vgId; + int32_t refCount; + int32_t vgVersion; + int8_t dropped; + int8_t accessState; + uint64_t dbUid; + char *db; + char *path; + SVnode *pImpl; + STaosQueue *pWriteQ; + STaosQueue *pSyncQ; + STaosQueue *pApplyQ; + STaosQueue *pQueryQ; + STaosQueue* pFetchQ; } SVnodeObj; typedef struct { @@ -72,9 +72,9 @@ static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); +static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); +static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); @@ -768,7 +768,7 @@ static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp); } -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -804,7 +804,7 @@ static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t taosArrayDestroy(pArray); } -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -815,7 +815,7 @@ static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t } } -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -826,7 +826,7 @@ static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t } } -static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { +static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) { int32_t code = 0; if (pQueue == NULL) { diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 04bc0c8dc8..75f5e9cdbc 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -14,26 +14,29 @@ */ #include "os.h" -#include "ulog.h" + #include "taoserror.h" #include "tqueue.h" +#include "ulog.h" + +typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { - struct STaosQnode *next; - char item[]; + STaosQnode *next; + char item[]; } STaosQnode; typedef struct STaosQueue { - int32_t itemSize; - int32_t numOfItems; - struct STaosQnode *head; - struct STaosQnode *tail; - struct STaosQueue *next; // for queue set - struct STaosQset *qset; // for queue set - void *ahandle; // for queue set - FProcessItem itemFp; - FProcessItems itemsFp; - pthread_mutex_t mutex; + int32_t itemSize; + int32_t numOfItems; + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FProcessItem itemFp; + FProcessItems itemsFp; + pthread_mutex_t mutex; } STaosQueue; typedef struct STaosQset { @@ -52,8 +55,8 @@ typedef struct STaosQall { int32_t numOfItems; } STaosQall; -taos_queue taosOpenQueue() { - STaosQueue *queue = (STaosQueue *)calloc(sizeof(STaosQueue), 1); +STaosQueue *taosOpenQueue() { + STaosQueue *queue = calloc(sizeof(STaosQueue), 1); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -65,16 +68,14 @@ taos_queue taosOpenQueue() { return queue; } -void taosSetQueueFp(taos_queue param, FProcessItem itemFp, FProcessItems itemsFp) { - if (param == NULL) return; - STaosQueue *queue = (STaosQueue *)param; +void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp) { + if (queue == NULL) return; queue->itemFp = itemFp; queue->itemsFp = itemsFp; } -void taosCloseQueue(taos_queue param) { - if (param == NULL) return; - STaosQueue *queue = (STaosQueue *)param; +void taosCloseQueue(STaosQueue *queue) { + if (queue == NULL) return; STaosQnode *pTemp; STaosQset *qset; @@ -98,9 +99,8 @@ void taosCloseQueue(taos_queue param) { uTrace("queue:%p is closed", queue); } -bool taosQueueEmpty(taos_queue param) { - if (param == NULL) return true; - STaosQueue *queue = (STaosQueue *)param; +bool taosQueueEmpty(STaosQueue *queue) { + if (queue == NULL) return true; bool empty = false; pthread_mutex_lock(&queue->mutex); @@ -112,7 +112,7 @@ bool taosQueueEmpty(taos_queue param) { return empty; } -void *taosAllocateQitem(int size) { +void *taosAllocateQitem(int32_t size) { STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1); if (pNode == NULL) return NULL; @@ -129,9 +129,8 @@ void taosFreeQitem(void *param) { free(temp); } -int taosWriteQitem(taos_queue param, void *item) { - STaosQueue *queue = (STaosQueue *)param; - STaosQnode *pNode = (STaosQnode *)(((char *)item) - sizeof(STaosQnode)); +int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { + STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode)); pNode->next = NULL; pthread_mutex_lock(&queue->mutex); @@ -146,7 +145,7 @@ int taosWriteQitem(taos_queue param, void *item) { queue->numOfItems++; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - uTrace("item:%p is put into queue:%p, items:%d", item, queue, queue->numOfItems); + uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); pthread_mutex_unlock(&queue->mutex); @@ -155,22 +154,21 @@ int taosWriteQitem(taos_queue param, void *item) { return 0; } -int taosReadQitem(taos_queue param, void **pitem) { - STaosQueue *queue = (STaosQueue *)param; +int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { STaosQnode *pNode = NULL; - int code = 0; + int32_t code = 0; pthread_mutex_lock(&queue->mutex); if (queue->head) { pNode = queue->head; - *pitem = pNode->item; + *ppItem = pNode->item; queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); code = 1; - uDebug("item:%p is read out from queue:%p, items:%d", *pitem, queue, queue->numOfItems); + uDebug("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); @@ -178,18 +176,13 @@ int taosReadQitem(taos_queue param, void **pitem) { return code; } -void *taosAllocateQall() { - void *p = calloc(sizeof(STaosQall), 1); - return p; -} +STaosQall *taosAllocateQall() { return calloc(sizeof(STaosQall), 1); } -void taosFreeQall(void *param) { free(param); } +void taosFreeQall(STaosQall *qall) { free(qall); } -int taosReadAllQitems(taos_queue param, taos_qall p2) { - STaosQueue *queue = (STaosQueue *)param; - STaosQall *qall = (STaosQall *)p2; - int code = 0; - bool empty; +int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { + int32_t code = 0; + bool empty; pthread_mutex_lock(&queue->mutex); @@ -219,29 +212,25 @@ int taosReadAllQitems(taos_queue param, taos_qall p2) { return code; } -int taosGetQitem(taos_qall param, void **pitem) { - STaosQall *qall = (STaosQall *)param; +int32_t taosGetQitem(STaosQall *qall, void **ppItem) { STaosQnode *pNode; - int num = 0; + int32_t num = 0; pNode = qall->current; if (pNode) qall->current = pNode->next; if (pNode) { - *pitem = pNode->item; + *ppItem = pNode->item; num = 1; - uTrace("item:%p is fetched", *pitem); + uTrace("item:%p is fetched", *ppItem); } return num; } -void taosResetQitems(taos_qall param) { - STaosQall *qall = (STaosQall *)param; - qall->current = qall->start; -} +void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } -taos_qset taosOpenQset() { +STaosQset *taosOpenQset() { STaosQset *qset = (STaosQset *)calloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -255,9 +244,8 @@ taos_qset taosOpenQset() { return qset; } -void taosCloseQset(taos_qset param) { - if (param == NULL) return; - STaosQset *qset = (STaosQset *)param; +void taosCloseQset(STaosQset *qset) { + if (qset == NULL) return; // remove all the queues from qset pthread_mutex_lock(&qset->mutex); @@ -279,16 +267,12 @@ void taosCloseQset(taos_qset param) { // tsem_post 'qset->sem', so that reader threads waiting for it // resumes execution and return, should only be used to signal the // thread to exit. -void taosQsetThreadResume(taos_qset param) { - STaosQset *qset = (STaosQset *)param; +void taosQsetThreadResume(STaosQset *qset) { uDebug("qset:%p, it will exit", qset); tsem_post(&qset->sem); } -int taosAddIntoQset(taos_qset p1, taos_queue p2, void *ahandle) { - STaosQueue *queue = (STaosQueue *)p2; - STaosQset *qset = (STaosQset *)p1; - +int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { if (queue->qset) return -1; pthread_mutex_lock(&qset->mutex); @@ -309,10 +293,7 @@ int taosAddIntoQset(taos_qset p1, taos_queue p2, void *ahandle) { return 0; } -void taosRemoveFromQset(taos_qset p1, taos_queue p2) { - STaosQueue *queue = (STaosQueue *)p2; - STaosQset *qset = (STaosQset *)p1; - +void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *tqueue = NULL; pthread_mutex_lock(&qset->mutex); @@ -353,18 +334,17 @@ void taosRemoveFromQset(taos_qset p1, taos_queue p2) { uTrace("queue:%p is removed from qset:%p", queue, qset); } -int taosGetQueueNumber(taos_qset param) { return ((STaosQset *)param)->numOfQueues; } +int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } -int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProcessItem *itemFp) { - STaosQset *qset = (STaosQset *)param; +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp) { STaosQnode *pNode = NULL; - int code = 0; + int32_t code = 0; tsem_wait(&qset->sem); pthread_mutex_lock(&qset->mutex); - for (int i = 0; i < qset->numOfQueues; ++i) { + for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; STaosQueue *queue = qset->current; if (queue) qset->current = queue->next; @@ -375,7 +355,7 @@ int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProces if (queue->head) { pNode = queue->head; - *pitem = pNode->item; + *ppItem = pNode->item; if (ahandle) *ahandle = queue->ahandle; if (itemFp) *itemFp = queue->itemFp; queue->head = pNode->next; @@ -383,7 +363,7 @@ int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProces queue->numOfItems--; atomic_sub_fetch_32(&qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *pitem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); @@ -395,18 +375,15 @@ int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProces return code; } -int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **ahandle, FProcessItems *itemsFp) { - STaosQset *qset = (STaosQset *)param; +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp) { STaosQueue *queue; - STaosQall *qall = (STaosQall *)p2; - int code = 0; + int32_t code = 0; tsem_wait(&qset->sem); pthread_mutex_lock(&qset->mutex); - for(int i=0; inumOfQueues; ++i) { - if (qset->current == NULL) - qset->current = qset->head; + for (int32_t i = 0; i < qset->numOfQueues; ++i) { + if (qset->current == NULL) qset->current = qset->head; queue = qset->current; if (queue) qset->current = queue->next; if (queue == NULL) break; @@ -427,34 +404,32 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **ahandle, FPr queue->tail = NULL; queue->numOfItems = 0; atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); - for (int j=1; jnumOfItems; ++j) tsem_wait(&qset->sem); - } + for (int32_t j = 1; j < qall->numOfItems; ++j) tsem_wait(&qset->sem); + } pthread_mutex_unlock(&queue->mutex); - if (code != 0) break; + if (code != 0) break; } pthread_mutex_unlock(&qset->mutex); return code; } -int taosGetQueueItemsNumber(taos_queue param) { - STaosQueue *queue = (STaosQueue *)param; +int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; - int num; + int32_t num; pthread_mutex_lock(&queue->mutex); num = queue->numOfItems; pthread_mutex_unlock(&queue->mutex); return num; } -int taosGetQsetItemsNumber(taos_qset param) { - STaosQset *qset = (STaosQset *)param; +int32_t taosGetQsetItemsNumber(STaosQset *qset) { if (!qset) return 0; - int num = 0; + int32_t num = 0; pthread_mutex_lock(&qset->mutex); num = qset->numOfItems; pthread_mutex_unlock(&qset->mutex); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index fb7b71b845..ed74041712 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -85,9 +85,9 @@ static void *tWorkerThreadFp(SWorker *worker) { return NULL; } -taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) { +STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) { pthread_mutex_lock(&pool->mutex); - taos_queue queue = taosOpenQueue(); + STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); return NULL; @@ -121,7 +121,7 @@ taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) return queue; } -void tWorkerFreeQueue(SWorkerPool *pool, void *queue) { +void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } @@ -195,11 +195,11 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) { return NULL; } -taos_queue tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) { +STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) { pthread_mutex_lock(&pool->mutex); SMWorker *worker = pool->workers + pool->nextId; - taos_queue *queue = taosOpenQueue(); + STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); return NULL; @@ -250,7 +250,7 @@ taos_queue tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems f return queue; } -void tMWorkerFreeQueue(SMWorkerPool *pool, taos_queue queue) { +void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } From e3090485135aa59fb2b98ea7aca6d6002a9a94df Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 30 Dec 2021 19:24:07 +0800 Subject: [PATCH 48/97] handle read/write crash concurrently --- source/libs/index/src/index_cache.c | 4 +- source/libs/index/test/indexTests.cc | 59 ++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index bce9e5fb87..0e46445a00 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -181,9 +181,9 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { break; } else if (cache->imm != NULL) { // TODO: wake up by condition variable - // pthread_mutex_unlock(&cache->mtx); + pthread_mutex_unlock(&cache->mtx); taosMsleep(50); - // pthread_mutex_lock(&cache->mtx); + pthread_mutex_lock(&cache->mtx); } else { indexCacheRef(cache); cache->imm = cache->mem; diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index cffb09ee4d..080becccf1 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include "index.h" #include "indexInt.h" #include "index_cache.h" @@ -25,6 +26,9 @@ #include "tskiplist.h" #include "tutil.h" using namespace std; + +#define NUM_OF_THREAD 10 + class DebugInfo { public: DebugInfo(const char* str) : info(str) { @@ -41,6 +45,7 @@ class DebugInfo { private: std::string info; }; + class FstWriter { public: FstWriter() { @@ -637,6 +642,23 @@ class IndexObj { indexMultiTermDestroy(terms); return numOfTable; } + int WriteMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world", + size_t numOfTable = 100 * 10000) { + std::string tColVal = colVal; + for (int i = 0; i < numOfTable; i++) { + tColVal[tColVal.size() - 1] = 'a' + i % 26; + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 10; i++) { + int ret = Put(terms, i); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + return numOfTable; + } int Put(SIndexMultiTerm* fvs, uint64_t uid) { numOfWrite += taosArrayGetSize(fvs); @@ -659,6 +681,14 @@ class IndexObj { return taosArrayGetSize(result); // assert(taosArrayGetSize(result) == targetSize); } + void PutOne(const std::string& colName, const std::string& colVal) { + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + Put(terms, 10); + indexMultiTermDestroy(terms); + } void Debug() { std::cout << "numOfWrite:" << numOfWrite << std::endl; std::cout << "numOfRead:" << numOfRead << std::endl; @@ -758,15 +788,38 @@ TEST_F(IndexEnv2, testIndexOpen) { TEST_F(IndexEnv2, testIndex_TrigeFlush) { std::string path = "/tmp/test"; - if (index->Init(path) != 0) {} + if (index->Init(path) != 0) { + // r + std::cout << "failed to init" << std::endl; + } int numOfTable = 100 * 10000; index->WriteMillonData("tag1", "Hello", numOfTable); int target = index->SearchOne("tag1", "Hello"); assert(numOfTable == target); } + +static void write_and_search(IndexObj* idx) { + std::string colName("tag1"), colVal("Hello"); + + int target = idx->SearchOne("tag1", "Hello"); + idx->PutOne(colName, colVal); +} TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + // opt + } + index->WriteMultiMillonData("tag1", "Hello", 200000); + std::thread threads[NUM_OF_THREAD]; + + for (int i = 0; i < NUM_OF_THREAD; i++) { + // + threads[i] = std::thread(write_and_search, index); + } + for (int i = 0; i < NUM_OF_THREAD; i++) { + // TOD + threads[i].join(); + } } TEST_F(IndexEnv2, testIndex_multi_thread_write) { std::string path = "/tmp"; From 4f3c0c5aaec13d26e580beada3c63a4fa1c25912 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 30 Dec 2021 11:28:24 +0000 Subject: [PATCH 49/97] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index aafbb1b4e7..af8af6a052 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -81,7 +81,7 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "meta.db", false) < 0) { + if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "schema.db", false) < 0) { metaCloseDB(pMeta); return -1; } @@ -558,6 +558,12 @@ SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { pDB->pTbDB->cursor(pDB->pTbDB, NULL, &(pTbCur->pCur), 0); +#if 0 + DB_BTREE_STAT *sp; + pDB->pTbDB->stat(pDB->pTbDB, NULL, &sp, 0); + printf("**************** %ld\n", sp->bt_nkeys); +#endif + return pTbCur; } From 31868e951a372683fe15d1a8b8dbebba1296bcdd Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Thu, 30 Dec 2021 20:55:44 +0800 Subject: [PATCH 50/97] add traft --- cmake/cmake.options | 6 + cmake/traft_CMakeLists.txt.in | 14 + contrib/CMakeLists.txt | 18 + contrib/test/CMakeLists.txt | 4 + contrib/test/traft/CMakeLists.txt | 7 + contrib/test/traft/clear.sh | 4 + contrib/test/traft/common.h | 36 ++ contrib/test/traft/help.txt | 18 + contrib/test/traft/raftMain.c | 659 ++++++++++++++++++++++++++++++ contrib/test/traft/raftServer.c | 222 ++++++++++ contrib/test/traft/raftServer.h | 68 +++ 11 files changed, 1056 insertions(+) create mode 100644 cmake/traft_CMakeLists.txt.in create mode 100644 contrib/test/traft/CMakeLists.txt create mode 100644 contrib/test/traft/clear.sh create mode 100644 contrib/test/traft/common.h create mode 100644 contrib/test/traft/help.txt create mode 100644 contrib/test/traft/raftMain.c create mode 100644 contrib/test/traft/raftServer.c create mode 100644 contrib/test/traft/raftServer.h diff --git a/cmake/cmake.options b/cmake/cmake.options index 44fa8c7e4b..faa45256fb 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -56,6 +56,12 @@ option( OFF ) +option( + BUILD_WITH_TRAFT + "If build with traft" + OFF +) + option( BUILD_DEPENDENCY_TESTS "If build dependency tests" diff --git a/cmake/traft_CMakeLists.txt.in b/cmake/traft_CMakeLists.txt.in new file mode 100644 index 0000000000..9b571b3666 --- /dev/null +++ b/cmake/traft_CMakeLists.txt.in @@ -0,0 +1,14 @@ + +# traft +ExternalProject_Add(traft + GIT_REPOSITORY https://github.com/taosdata/traft.git + GIT_TAG for_3.0 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/traft" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft" + #BUILD_IN_SOURCE TRUE + # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example + BUILD_COMMAND "$(MAKE)" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c08f894fe7..074014b0d7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -41,6 +41,12 @@ if(${BUILD_WITH_CRAFT}) SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE) endif(${BUILD_WITH_CRAFT}) +# traft +if(${BUILD_WITH_TRAFT}) + cat("${CMAKE_SUPPORT_DIR}/traft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + SET(BUILD_WITH_UV ON CACHE BOOL "traft need libuv" FORCE) +endif(${BUILD_WITH_TRAFT}) + #libuv if(${BUILD_WITH_UV}) cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -173,6 +179,18 @@ if(${BUILD_WITH_CRAFT}) # ) endif(${BUILD_WITH_CRAFT}) +# TRAFT +if(${BUILD_WITH_TRAFT}) + add_library(traft STATIC IMPORTED GLOBAL) + set_target_properties(traft PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/traft/.libs/libraft.a" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/traft/include" + ) + # target_link_libraries(craft + # INTERFACE pthread + # ) +endif(${BUILD_WITH_TRAFT}) + # LIBUV if(${BUILD_WITH_UV}) add_subdirectory(libuv) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 330fe8f70f..0c71113056 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -19,4 +19,8 @@ if(${BUILD_WITH_CRAFT}) add_subdirectory(craft) endif(${BUILD_WITH_CRAFT}) +if(${BUILD_WITH_TRAFT}) + add_subdirectory(traft) +endif(${BUILD_WITH_TRAFT}) + add_subdirectory(tdev) diff --git a/contrib/test/traft/CMakeLists.txt b/contrib/test/traft/CMakeLists.txt new file mode 100644 index 0000000000..e29fea04f1 --- /dev/null +++ b/contrib/test/traft/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(raftMain "") +target_sources(raftMain + PRIVATE + "raftMain.c" + "raftServer.c" +) +target_link_libraries(raftMain PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/clear.sh b/contrib/test/traft/clear.sh new file mode 100644 index 0000000000..398b3088f2 --- /dev/null +++ b/contrib/test/traft/clear.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -rf 127.0.0.1* +rm -rf ./data diff --git a/contrib/test/traft/common.h b/contrib/test/traft/common.h new file mode 100644 index 0000000000..0229c29cf7 --- /dev/null +++ b/contrib/test/traft/common.h @@ -0,0 +1,36 @@ +#ifndef TDENGINE_COMMON_H +#define TDENGINE_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define MAX_INSTANCE_NUM 100 + +#define MAX_PEERS 10 +#define COMMAND_LEN 1024 +#define TOKEN_LEN 128 +#define DIR_LEN 256 +#define HOST_LEN 64 +#define ADDRESS_LEN (HOST_LEN + 16) + +typedef struct { + char host[HOST_LEN]; + uint32_t port; +} Addr; + +typedef struct { + Addr me; + Addr peers[MAX_PEERS]; + int peersCount; + char dir[DIR_LEN]; + char dataDir[DIR_LEN + HOST_LEN * 2]; +} SRaftServerConfig; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_COMMON_H diff --git a/contrib/test/traft/help.txt b/contrib/test/traft/help.txt new file mode 100644 index 0000000000..7709e80e0a --- /dev/null +++ b/contrib/test/traft/help.txt @@ -0,0 +1,18 @@ + +make raftServer + +all: + gcc raftMain.c raftServer.c -I ../../traft/include/ ../../traft/.libs/libraft.a -o raftMain -luv -llz4 -lpthread -g +clean: + rm -f raftMain + sh clear.sh + + +make traft: + +sudo apt-get install libuv1-dev liblz4-dev +autoreconf -i +./configure --enable-example +make + + diff --git a/contrib/test/traft/raftMain.c b/contrib/test/traft/raftMain.c new file mode 100644 index 0000000000..24ad93856c --- /dev/null +++ b/contrib/test/traft/raftMain.c @@ -0,0 +1,659 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "raftServer.h" +#include "common.h" + +const char *exe_name; + +typedef struct LeaderState { + char address[48]; + int leaderCount; + +} LeaderState; + +#define NODE_COUNT 3 +LeaderState leaderStates[NODE_COUNT]; + +void printLeaderCount() { + for (int i = 0; i < NODE_COUNT; ++i) { + printf("%s: leaderCount:%d \n", leaderStates[i].address, leaderStates[i].leaderCount); + } +} + +void updateLeaderStates(SRaftServer *pRaftServer) { + for (int i = 0; i < pRaftServer->instance[0].raft.configuration.n; ++i) { + snprintf(leaderStates[i].address, sizeof(leaderStates[i].address), "%s", pRaftServer->instance[0].raft.configuration.servers[i].address); + leaderStates[i].leaderCount = 0; + } + + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + struct raft *r = &pRaftServer->instance[i].raft; + + char leaderAddress[128]; + memset(leaderAddress, 0, sizeof(leaderAddress)); + + if (r->state == RAFT_LEADER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); + } else if (r->state == RAFT_FOLLOWER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); + } + + for (int j = 0; j < NODE_COUNT; j++) { + if (strcmp(leaderAddress, leaderStates[j].address) == 0) { + leaderStates[j].leaderCount++; + } + } + } +} + + +void raftTransferCb(struct raft_transfer *req) { + SRaftServer *pRaftServer = req->data; + raft_free(req); + + printf("raftTransferCb: \n"); + updateLeaderStates(pRaftServer); + printLeaderCount(); + + int myLeaderCount; + for (int i = 0; i < NODE_COUNT; ++i) { + if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { + myLeaderCount = leaderStates[i].leaderCount; + } + } + + printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); + if (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { + struct raft *r; + for (int j = 0; j < pRaftServer->instanceCount; ++j) { + if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { + r = &pRaftServer->instance[j].raft; + } + } + + struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); + transfer->data = pRaftServer; + + uint64_t destRaftId; + int minIndex = -1; + int minLeaderCount = myLeaderCount; + for (int j = 0; j < NODE_COUNT; ++j) { + if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; + if (leaderStates[j].leaderCount <= minLeaderCount) { + minIndex = j; + } + } + + char myHost[48]; + uint16_t myPort; + uint16_t myVid; + decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); + + char *destAddress = leaderStates[minIndex].address; + + char tokens[MAX_PEERS][MAX_TOKEN_LEN]; + splitString(destAddress, ":", tokens, 2); + char *destHost = tokens[0]; + uint16_t destPort = atoi(tokens[1]); + destRaftId = encodeRaftId(destHost, destPort, myVid); + + raft_transfer(r, transfer, destRaftId, raftTransferCb); + } + +} + + +void parseAddr(const char *addr, char *host, int len, uint32_t *port) { + char* tmp = (char*)malloc(strlen(addr) + 1); + strcpy(tmp, addr); + + char* context; + char* separator = ":"; + char* token = strtok_r(tmp, separator, &context); + if (token) { + snprintf(host, len, "%s", token); + } + + token = strtok_r(NULL, separator, &context); + if (token) { + sscanf(token, "%u", port); + } + + free(tmp); +} + +// only parse 3 tokens +int parseCommand3(const char* str, char* token1, char* token2, char* token3, int len) +{ + char* tmp = (char*)malloc(strlen(str) + 1); + strcpy(tmp, str); + + char* context; + char* separator = " "; + int n = 0; + + char* token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token1, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token2, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token3, token, len); + n++; + } + +ret: + return n; + free(tmp); +} + +// only parse 4 tokens +int parseCommand4(const char* str, char* token1, char* token2, char* token3, char *token4, int len) +{ + char* tmp = (char*)malloc(strlen(str) + 1); + strcpy(tmp, str); + + char* context; + char* separator = " "; + int n = 0; + + char* token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token1, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token2, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token3, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token4, token, len); + n++; + } + +ret: + return n; + free(tmp); +} + +void *startServerFunc(void *param) { + SRaftServer *pServer = (SRaftServer*)param; + int32_t r = raftServerStart(pServer); + assert(r == 0); + + return NULL; +} + +// Console --------------------------------- +const char* state2String(unsigned short state) { + if (state == RAFT_UNAVAILABLE) { + return "RAFT_UNAVAILABLE"; + + } else if (state == RAFT_FOLLOWER) { + return "RAFT_FOLLOWER"; + + } else if (state == RAFT_CANDIDATE) { + return "RAFT_CANDIDATE"; + + } else if (state == RAFT_LEADER) { + return "RAFT_LEADER"; + + } + return "UNKNOWN_RAFT_STATE"; +} + + +void printRaftState2(struct raft *r) { + + char leaderAddress[128]; + memset(leaderAddress, 0, sizeof(leaderAddress)); + + if (r->state == RAFT_LEADER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); + } else if (r->state == RAFT_FOLLOWER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); + } + + for (int i = 0; i < r->configuration.n; ++i) { + char tmpAddress[128]; + snprintf(tmpAddress, sizeof(tmpAddress), "%s", r->configuration.servers[i].address); + + uint64_t raftId = r->configuration.servers[i].id; + char host[128]; + uint16_t port; + uint16_t vid; + decodeRaftId(raftId, host, 128, &port, &vid); + + char buf[512]; + memset(buf, 0, sizeof(buf)); + if (strcmp(tmpAddress, leaderAddress) == 0) { + snprintf(buf, sizeof(buf), "<%s:%u-%u-LEADER>\t", host, port, vid); + } else { + snprintf(buf, sizeof(buf), "<%s:%u-%u-FOLLOWER>\t", host, port, vid); + } + printf("%s", buf); + } + printf("\n"); +} + +void printRaftConfiguration(struct raft_configuration *c) { + printf("configuration: \n"); + for (int i = 0; i < c->n; ++i) { + printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); + } +} + +void printRaftState(struct raft *r) { + printf("----Raft State: -----------\n"); + printf("mem_addr: %p \n", r); + printf("my_id: %llu \n", r->id); + printf("address: %s \n", r->address); + printf("current_term: %llu \n", r->current_term); + printf("voted_for: %llu \n", r->voted_for); + printf("role: %s \n", state2String(r->state)); + printf("commit_index: %llu \n", r->commit_index); + printf("last_applied: %llu \n", r->last_applied); + printf("last_stored: %llu \n", r->last_stored); + + printf("configuration_index: %llu \n", r->configuration_index); + printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); + printRaftConfiguration(&r->configuration); + + printf("----------------------------\n"); +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + raft_free(req); + struct raft *r = req->data; + if (status != 0) { + printf("putValueCb: %s \n", raft_errmsg(r)); + } else { + printf("putValueCb: %s \n", "ok"); + } +} + +void putValue(struct raft *r, const char *value) { + struct raft_buffer buf; + + buf.len = TOKEN_LEN;; + buf.base = raft_malloc(buf.len); + snprintf(buf.base, buf.len, "%s", value); + + struct raft_apply *req = raft_malloc(sizeof(struct raft_apply)); + req->data = r; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char*)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +void getValue(const char *key) { + char *ptr = getKV(key); + if (ptr) { + printf("get value: [%s] \n", ptr); + } else { + printf("value not found for key: [%s] \n", key); + } +} + +void console(SRaftServer *pRaftServer) { + while (1) { + char cmd_buf[COMMAND_LEN]; + memset(cmd_buf, 0, sizeof(cmd_buf)); + char *ret = fgets(cmd_buf, COMMAND_LEN, stdin); + if (!ret) { + exit(-1); + } + + int pos = strlen(cmd_buf); + if(cmd_buf[pos - 1] == '\n') { + cmd_buf[pos - 1] = '\0'; + } + + if (strncmp(cmd_buf, "", COMMAND_LEN) == 0) { + continue; + } + + char cmd[TOKEN_LEN]; + memset(cmd, 0, sizeof(cmd)); + + char param1[TOKEN_LEN]; + memset(param1, 0, sizeof(param1)); + + char param2[TOKEN_LEN]; + memset(param2, 0, sizeof(param2)); + + char param3[TOKEN_LEN]; + memset(param2, 0, sizeof(param2)); + + parseCommand4(cmd_buf, cmd, param1, param2, param3, TOKEN_LEN); + if (strcmp(cmd, "addnode") == 0) { + printf("not support \n"); + + /* + char host[HOST_LEN]; + uint32_t port; + parseAddr(param1, host, HOST_LEN, &port); + uint64_t rid = raftId(host, port); + + struct raft_change *req = raft_malloc(sizeof(*req)); + int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL); + if (r != 0) { + printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft)); + } + printf("add node: %lu %s \n", rid, param1); + + struct raft_change *req2 = raft_malloc(sizeof(*req2)); + r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL); + if (r != 0) { + printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft)); + } + */ + + } else if (strcmp(cmd, "dropnode") == 0) { + printf("not support \n"); + + } else if (strcmp(cmd, "rebalance") == 0) { + + /* + updateLeaderStates(pRaftServer); + + int myLeaderCount; + for (int i = 0; i < NODE_COUNT; ++i) { + if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { + myLeaderCount = leaderStates[i].leaderCount; + } + } + + while (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { + printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); + + struct raft *r; + for (int j = 0; j < pRaftServer->instanceCount; ++j) { + if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { + r = &pRaftServer->instance[j].raft; + } + } + + struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); + transfer->data = pRaftServer; + + uint64_t destRaftId; + int minIndex = -1; + int minLeaderCount = myLeaderCount; + for (int j = 0; j < NODE_COUNT; ++j) { + if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; + + printf("-----leaderStates[%d].leaderCount:%d \n", j, leaderStates[j].leaderCount); + if (leaderStates[j].leaderCount <= minLeaderCount) { + minIndex = j; + printf("++++ assign minIndex : %d \n", minIndex); + } + } + + printf("minIndex:%d minLeaderCount:%d \n", minIndex, minLeaderCount); + + char myHost[48]; + uint16_t myPort; + uint16_t myVid; + decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); + + char *destAddress = leaderStates[minIndex].address; + + char tokens[MAX_PEERS][MAX_TOKEN_LEN]; + splitString(destAddress, ":", tokens, 2); + char *destHost = tokens[0]; + uint16_t destPort = atoi(tokens[1]); + destRaftId = encodeRaftId(destHost, destPort, myVid); + + printf("destHost:%s destPort:%u myVid:%u", destHost, destPort, myVid); + raft_transfer(r, transfer, destRaftId, raftTransferCb); + sleep(1); + + for (int i = 0; i < NODE_COUNT; ++i) { + if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { + myLeaderCount = leaderStates[i].leaderCount; + } + } + } + */ + + + int leaderCount = 0; + + struct raft *firstR; + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + struct raft *r = &pRaftServer->instance[i].raft; + if (r->state == RAFT_LEADER) { + leaderCount++; + firstR = r; + } + } + + if (leaderCount > pRaftServer->instanceCount / NODE_COUNT) { + struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); + transfer->data = pRaftServer; + raft_transfer(firstR, transfer, 0, raftTransferCb); + } + + + } else if (strcmp(cmd, "put") == 0) { + char buf[256]; + uint16_t vid; + sscanf(param1, "%hu", &vid); + snprintf(buf, sizeof(buf), "%s--%s", param2, param3); + putValue(&pRaftServer->instance[vid].raft, buf); + + } else if (strcmp(cmd, "get") == 0) { + getValue(param1); + + } else if (strcmp(cmd, "transfer") == 0) { + uint16_t vid; + sscanf(param1, "%hu", &vid); + + struct raft_transfer transfer; + raft_transfer(&pRaftServer->instance[vid].raft, &transfer, 0, NULL); + + + } else if (strcmp(cmd, "state") == 0) { + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + printf("instance %d: ", i); + printRaftState(&pRaftServer->instance[i].raft); + } + + } else if (strcmp(cmd, "state2") == 0) { + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + printRaftState2(&pRaftServer->instance[i].raft); + } + + } else if (strcmp(cmd, "snapshot") == 0) { + printf("not support \n"); + + } else if (strcmp(cmd, "help") == 0) { + printf("addnode \"127.0.0.1:8888\" \n"); + printf("dropnode \"127.0.0.1:8888\" \n"); + printf("put key value \n"); + printf("get key \n"); + printf("state \n"); + + } else { + printf("unknown command: [%s], type \"help\" to see help \n", cmd); + } + + //printf("cmd_buf: [%s] \n", cmd_buf); + } +} + +void *startConsoleFunc(void *param) { + SRaftServer *pServer = (SRaftServer*)param; + console(pServer); + return NULL; +} + +// Config --------------------------------- +void usage() { + printf("\nusage: \n"); + printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name); + printf("\n"); + printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10002 --peers=127.0.0.1:10000,127.0.0.1:10001 --dir=./data \n", exe_name); + printf("\n"); +} + +void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { + memset(pConf, 0, sizeof(*pConf)); + + int option_index, option_value; + option_index = 0; + static struct option long_options[] = { + {"help", no_argument, NULL, 'h'}, + {"peers", required_argument, NULL, 'p'}, + {"me", required_argument, NULL, 'm'}, + {"dir", required_argument, NULL, 'd'}, + {NULL, 0, NULL, 0} + }; + + while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) { + switch (option_value) { + case 'm': { + parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); + break; + } + + case 'p': { + char tokens[MAX_PEERS][MAX_TOKEN_LEN]; + int peerCount = splitString(optarg, ",", tokens, MAX_PEERS); + pConf->peersCount = peerCount; + for (int i = 0; i < peerCount; ++i) { + Addr *pAddr = &pConf->peers[i]; + parseAddr(tokens[i], pAddr->host, sizeof(pAddr->host), &pAddr->port); + } + break; + } + + + case 'd': { + snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg); + break; + } + + case 'h': { + usage(); + exit(-1); + } + + default: { + usage(); + exit(-1); + } + } + } + snprintf(pConf->dataDir, sizeof(pConf->dataDir), "%s/%s_%u", pConf->dir, pConf->me.host, pConf->me.port); +} + +void printConf(SRaftServerConfig *pConf) { + printf("\nconf: \n"); + printf("me: %s:%u \n", pConf->me.host, pConf->me.port); + printf("peersCount: %d \n", pConf->peersCount); + for (int i = 0; i < pConf->peersCount; ++i) { + Addr *pAddr = &pConf->peers[i]; + printf("peer%d: %s:%u \n", i, pAddr->host, pAddr->port); + } + printf("dataDir: %s \n\n", pConf->dataDir); + +} + + +int main(int argc, char **argv) { + srand(time(NULL)); + int32_t ret; + + exe_name = argv[0]; + if (argc < 3) { + usage(); + exit(-1); + } + + SRaftServerConfig conf; + parseConf(argc, argv, &conf); + printConf(&conf); + + signal(SIGPIPE, SIG_IGN); + + /* + char cmd_buf[COMMAND_LEN]; + snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", conf.dataDir); + system(cmd_buf); + */ + + + struct raft_fsm fsm; + initFsm(&fsm); + + SRaftServer raftServer; + ret = raftServerInit(&raftServer, &conf, &fsm); + assert(ret == 0); + + pthread_t tidRaftServer; + pthread_create(&tidRaftServer, NULL, startServerFunc, &raftServer); + + pthread_t tidConsole; + pthread_create(&tidConsole, NULL, startConsoleFunc, &raftServer); + + while (1) { + sleep(10); + } + + return 0; +} diff --git a/contrib/test/traft/raftServer.c b/contrib/test/traft/raftServer.c new file mode 100644 index 0000000000..94de49cd0f --- /dev/null +++ b/contrib/test/traft/raftServer.c @@ -0,0 +1,222 @@ +#include +#include +#include "common.h" +#include "raftServer.h" + +char *keys; +char *values; + +void initStore() { + keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); + values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); + writeIndex = 0; +} + +void destroyStore() { + free(keys); + free(values); +} + +void putKV(const char *key, const char *value) { + if (writeIndex < MAX_RECORD_COUNT) { + strncpy(&keys[writeIndex], key, MAX_KV_LEN); + strncpy(&values[writeIndex], value, MAX_KV_LEN); + writeIndex++; + } +} + +char *getKV(const char *key) { + for (int i = 0; i < MAX_RECORD_COUNT; ++i) { + if (strcmp(&keys[i], key) == 0) { + return &values[i]; + } + } + return NULL; +} + + +int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr) +{ + if (n_arr <= 0) { + return -1; + } + + char* tmp = (char*)malloc(strlen(str) + 1); + strcpy(tmp, str); + char* context; + int n = 0; + + char* token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + strncpy(arr[n], token, MAX_TOKEN_LEN); + n++; + + while (1) { + token = strtok_r(NULL, separator, &context); + if (!token || n >= n_arr) { + goto ret; + } + strncpy(arr[n], token, MAX_TOKEN_LEN); + n++; + } + +ret: + free(tmp); + return n; +} + +/* +uint64_t raftId(const char *host, uint32_t port) { + uint32_t host_uint32 = (uint32_t)inet_addr(host); + assert(host_uint32 != (uint32_t)-1); + uint64_t code = ((uint64_t)host_uint32) << 32 | port; + return code; +} +*/ + + +/* +uint64_t encodeRaftId(const char *host, uint16_t port, uint16_t vid) { + uint64_t raftId; + uint32_t host_uint32 = (uint32_t)inet_addr(host); + assert(host_uint32 != (uint32_t)-1); + + raftId = (((uint64_t)host_uint32) << 32) | (((uint32_t)port) << 16) | vid; + return raftId; +} + +void decodeRaftId(uint64_t raftId, char *host, int32_t len, uint16_t *port, uint16_t *vid) { + uint32_t host32 = (uint32_t)((raftId >> 32) & 0x00000000FFFFFFFF); + + struct in_addr addr; + addr.s_addr = host32; + snprintf(host, len, "%s", inet_ntoa(addr)); + + *port = (uint16_t)((raftId & 0x00000000FFFF0000) >> 16); + *vid = (uint16_t)(raftId & 0x000000000000FFFF); +} +*/ + + + + +int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm) { + int ret; + + snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); + pRaftServer->port = pConf->me.port; + snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); + //strncpy(pRaftServer->dir, pConf->dataDir, sizeof(pRaftServer->dir)); + + ret = uv_loop_init(&pRaftServer->loop); + if (ret != 0) { + fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); + assert(0); + } + + ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); + if (ret != 0) { + fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); + assert(0); + } + + + uint16_t vid; + pRaftServer->instanceCount = 20; + + + for (int i = 0; i < pRaftServer->instanceCount; ++i) + { + //vid = 0; + vid = i; + + + pRaftServer->instance[vid].raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); + snprintf(pRaftServer->instance[vid].dir, sizeof(pRaftServer->instance[vid].dir), "%s_%llu", pConf->dataDir, pRaftServer->instance[vid].raftId); + + char cmd_buf[COMMAND_LEN]; + snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", pRaftServer->instance[vid].dir); + system(cmd_buf); + sleep(1); + + pRaftServer->instance[vid].fsm = pFsm; + + ret = raft_uv_init(&pRaftServer->instance[vid].io, &pRaftServer->loop, pRaftServer->instance[vid].dir, &pRaftServer->transport); + if (ret != 0) { + fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); + assert(0); + } + + ret = raft_init(&pRaftServer->instance[vid].raft, &pRaftServer->instance[vid].io, pRaftServer->instance[vid].fsm, pRaftServer->instance[vid].raftId, pRaftServer->address); + if (ret != 0) { + fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); + assert(0); + } + + struct raft_configuration conf; + raft_configuration_init(&conf); + raft_configuration_add(&conf, pRaftServer->instance[vid].raftId, pRaftServer->address, RAFT_VOTER); + printf("add myself: %llu - %s \n", pRaftServer->instance[vid].raftId, pRaftServer->address); + for (int i = 0; i < pConf->peersCount; ++i) { + const Addr *pAddr = &pConf->peers[i]; + + raft_id rid = encodeRaftId(pAddr->host, pAddr->port, vid); + + char addrBuf[ADDRESS_LEN]; + snprintf(addrBuf, sizeof(addrBuf), "%s:%u", pAddr->host, pAddr->port); + raft_configuration_add(&conf, rid, addrBuf, RAFT_VOTER); + printf("add peers: %llu - %s \n", rid, addrBuf); + } + + raft_bootstrap(&pRaftServer->instance[vid].raft, &conf); + + } + + + + + + + + return 0; +} + +int32_t raftServerStart(SRaftServer *pRaftServer) { + int ret; + + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + ret = raft_start(&pRaftServer->instance[i].raft); + if (ret != 0) { + fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[i].raft)); + } + + } + + + uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); +} + + +void raftServerClose(SRaftServer *pRaftServer) { + +} + + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { + char *msg = (char*)buf->base; + printf("fsm apply: %s \n", msg); + + char arr[2][MAX_TOKEN_LEN]; + splitString(msg, "--", arr, 2); + putKV(arr[0], arr[1]); + + return 0; +} + +int32_t initFsm(struct raft_fsm *fsm) { + initStore(); + fsm->apply = fsmApplyCb; + return 0; +} diff --git a/contrib/test/traft/raftServer.h b/contrib/test/traft/raftServer.h new file mode 100644 index 0000000000..b1f62caac5 --- /dev/null +++ b/contrib/test/traft/raftServer.h @@ -0,0 +1,68 @@ +#ifndef TDENGINE_RAFT_SERVER_H +#define TDENGINE_RAFT_SERVER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include "raft.h" +#include "raft/uv.h" +#include "common.h" + + +// simulate a db store, just for test +#define MAX_KV_LEN 100 +#define MAX_RECORD_COUNT 500 +char *keys; +char *values; +int writeIndex; + +void initStore(); +void destroyStore(); +void putKV(const char *key, const char *value); +char *getKV(const char *key); + +typedef struct { + char dir[DIR_LEN + HOST_LEN * 4]; /* Data dir of UV I/O backend */ + raft_id raftId; /* For vote */ + struct raft_fsm *fsm; /* Sample application FSM */ + struct raft raft; /* Raft instance */ + struct raft_io io; /* UV I/O backend */ + +} SInstance; + +typedef struct { + char host[HOST_LEN]; + uint32_t port; + char address[ADDRESS_LEN]; /* Raft instance address */ + + struct uv_loop_s loop; /* UV loop */ + struct raft_uv_transport transport; /* UV I/O backend transport */ + + SInstance instance[MAX_INSTANCE_NUM]; + int32_t instanceCount; + +} SRaftServer; + +#define MAX_TOKEN_LEN 32 +int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr); + +int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm); +int32_t raftServerStart(SRaftServer *pRaftServer); +void raftServerClose(SRaftServer *pRaftServer); + + +int initFsm(struct raft_fsm *fsm); + + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_RAFT_SERVER_H From 2eb0053e5ad173ced8ca6817ecc67ccfe52401c4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 30 Dec 2021 22:19:56 +0800 Subject: [PATCH 51/97] handle read/write crash concurrently --- include/util/tfile.h | 3 ++- source/libs/index/src/index.c | 2 +- source/libs/index/src/index_fst.c | 18 +++++++------ .../index/src/index_fst_counting_writer.c | 19 ++++++------- source/libs/index/src/index_tfile.c | 11 +++++--- source/libs/index/test/fstTest.cc | 2 +- source/util/src/tfile.c | 27 ++++++++++++------- 7 files changed, 50 insertions(+), 32 deletions(-) diff --git a/include/util/tfile.h b/include/util/tfile.h index af4c19e7d1..b3d141c443 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -38,6 +38,7 @@ int64_t tfOpenCreateWriteAppend(const char *pathname); int64_t tfClose(int64_t tfd); int64_t tfWrite(int64_t tfd, void *buf, int64_t count); int64_t tfRead(int64_t tfd, void *buf, int64_t count); +int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); int32_t tfFsync(int64_t tfd); bool tfValid(int64_t tfd); int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); @@ -47,4 +48,4 @@ int32_t tfFtruncate(int64_t tfd, int64_t length); } #endif -#endif /*_TD_UTIL_FILE_H*/ +#endif /*_TD_UTIL_FILE_H*/ diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 167d33f51f..5167196031 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -459,7 +459,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { } else { if (value->val != NULL) { taosArrayClear(value->val); } } - free(value->colVal); + // free(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 246f50a12b..04a08dafd2 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1033,15 +1033,17 @@ FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { } FstNode* fstGetRoot(Fst* fst) { - // pthread_mutex_lock(&fst->mtx); - if (fst->root != NULL) { - // pthread_mutex_unlock(&fst->mtx); - return fst->root; - } CompiledAddr rAddr = fstGetRootAddr(fst); - fst->root = fstGetNode(fst, rAddr); - // pthread_mutex_unlock(&fst->mtx); - return fst->root; + return fstGetNode(fst, rAddr); + // pthread_mutex_lock(&fst->mtx); + // if (fst->root != NULL) { + // // pthread_mutex_unlock(&fst->mtx); + // return fst->root; + //} + // CompiledAddr rAddr = fstGetRootAddr(fst); + // fst->root = fstGetNode(fst, rAddr); + //// pthread_mutex_unlock(&fst->mtx); + // return fst->root; } FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 710db563d9..7906dfea11 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -42,8 +42,8 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - tfLseek(ctx->file.fd, offset, 0); - nRead = tfRead(ctx->file.fd, buf, len); + // tfLseek(ctx->file.fd, offset, 0); + nRead = tfPread(ctx->file.fd, buf, len, offset); } else { // refactor later assert(0); @@ -52,6 +52,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { + // taosFsyncFile(ctx->file.fd); tfFsync(ctx->file.fd); // tfFlush(ctx->file.fd); } else { @@ -69,13 +70,15 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { + // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenCreateWriteAppend(path); } else { - ctx->file.fd = tfOpenReadWrite(path); + // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.fd = tfOpenRead(path); } memcpy(ctx->file.buf, path, strlen(path)); if (ctx->file.fd < 0) { - indexError("open file error %d", errno); + indexError("failed to open file, error %d", errno); goto END; } } else if (ctx->type == TMemory) { @@ -101,10 +104,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { tfClose(ctx->file.fd); - if (remove) { - indexError("rm file %s", ctx->file.buf); - unlink(ctx->file.buf); - } + if (remove) { unlink(ctx->file.buf); } } free(ctx); } @@ -144,7 +144,8 @@ int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) } uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } -int fstCountingWriterFlush(FstCountingWriter* write) { + +int fstCountingWriterFlush(FstCountingWriter* write) { WriterCtx* ctx = write->wrt; ctx->flush(ctx); // write->wtr->flush diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 177132c67d..fc4f8593a1 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -149,7 +149,6 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { // T_REF_INC(reader); reader->ctx = ctx; - if (0 != tfileReaderLoadHeader(reader)) { tfileReaderDestroy(reader); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, @@ -542,8 +541,14 @@ static int tfileReaderLoadHeader(TFileReader* reader) { char buf[TFILE_HEADER_SIZE] = {0}; int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); - assert(nread == sizeof(buf)); + if (nread == -1) { + // + indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.fd, reader->ctx->file.buf); + } + // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); + return 0; } static int tfileReaderLoadFst(TFileReader* reader) { @@ -576,7 +581,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* char* buf = calloc(1, total); if (buf == NULL) { return -1; } - nread = ctx->read(ctx, buf, total); + nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); assert(total == nread); for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 3b5410547d..da974ce6c4 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -189,7 +189,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 8417, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 313f1d97af..4cb20802c7 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -16,21 +16,19 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "ulog.h" -#include "tutil.h" #include "tref.h" +#include "tutil.h" +#include "ulog.h" static int32_t tsFileRsetId = -1; static int8_t tfInited = 0; -static void tfCloseFile(void *p) { - taosCloseFile((int32_t)(uintptr_t)p); -} +static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } int32_t tfInit() { int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if(old == 1) return 0; + if (old == 1) return 0; tsFileRsetId = taosOpenRef(2000, tfCloseFile); if (tsFileRsetId > 0) { return 0; @@ -79,9 +77,7 @@ int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode return tfOpenImp(fd); } -int64_t tfClose(int64_t tfd) { - return taosRemoveRef(tsFileRsetId, tfd); -} +int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); @@ -109,6 +105,19 @@ int64_t tfRead(int64_t tfd, void *buf, int64_t count) { return ret; } +int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + + int64_t ret = pread(fd, buf, count, offset); + if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); + + taosReleaseRef(tsFileRsetId, tfd); + return ret; +} + int32_t tfFsync(int64_t tfd) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; From 9859acf6eccaccbc986731305cff3d0d1f50f583 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Dec 2021 22:56:30 +0800 Subject: [PATCH 52/97] [td-11818] fix compiler warning. --- source/client/test/clientTests.cpp | 28 ++++++++++++++++++++++++++ source/libs/parser/inc/astGenerator.h | 1 - source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/astGenerator.c | 1 - source/libs/parser/src/dCDAstProcess.c | 17 ---------------- source/libs/parser/src/parserUtil.c | 9 +++++++-- 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 013cf794e3..f2c79bfd5a 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -462,3 +462,31 @@ TEST(testCase, show_table_Test) { taos_free_result(pRes); taos_close(pConn); } + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1) t_3 using st2 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 0febc5ea33..22806969af 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -125,7 +125,6 @@ typedef struct SCreatedTableInfo { SArray *pTagNames; // create by using super table, tag name SArray *pTagVals; // create by using super table, tag value char *fullname; // table full name - STagData tagdata; // true tag data, super table full name is in STagData int8_t igExist; // ignore if exists } SCreatedTableInfo; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index bf59a0f80a..789f5a98e2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -188,7 +188,7 @@ cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB // And "ids" is an identifer-or-string. %type ids {SToken} ids(A) ::= ID(X). {A = X; } -ids(A) ::= STRING(X). {A = X; } +//ids(A) ::= STRING(X). {A = X; } %type ifexists {SToken} ifexists(X) ::= IF EXISTS. { X.n = 1;} diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 3d12f0f3b7..9122b0df96 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -692,7 +692,6 @@ void freeCreateTableInfo(void* p) { taosArrayDestroy(pInfo->pTagNames); taosArrayDestroyEx(pInfo->pTagVals, freeItem); tfree(pInfo->fullname); - tfree(pInfo->tagdata.data); } SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type) { diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6aa4ad8275..fd99cb6f66 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -363,7 +363,6 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p // too long tag values will return invalid sql, not be truncated automatically SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); STableComInfo tinfo = getTableInfo(pSuperTableMeta); - STagData* pTag = &pCreateTableInfo->tagdata; SKVRowBuilder kvRowBuilder = {0}; if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { @@ -463,22 +462,6 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p SSchema* pSchema = &pTagSchema[i]; SToken* pItem = taosArrayGet(pValList, i); - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->n > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return buildInvalidOperationMsg(pMsgBuf, msg3); - } - } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - // if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - //// code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); - // if (code != TSDB_CODE_SUCCESS) { - // return buildInvalidOperationMsg(pMsgBuf, msg4); - // } - // } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { - // pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision); - // } - } - char tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0}; SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema}; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 6c7ecbe0ed..22545255a3 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -571,7 +571,9 @@ void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t fla pColumn->info.type = pSchema->type; if (tableName != NULL) { - snprintf(pColumn->name, tListLen(pColumn->name), "%s.%s", tableName, pSchema->name); + char n[TSDB_COL_NAME_LEN + 1 + TSDB_TABLE_NAME_LEN] = {0}; + snprintf(n, tListLen(n), "%s.%s", tableName, pSchema->name); + tstrncpy(pColumn->name, n, tListLen(pColumn->name)); } else { tstrncpy(pColumn->name, pSchema->name, tListLen(pColumn->name)); } @@ -586,7 +588,10 @@ SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSc c.info.type = pSchema->type; if (tableName != NULL) { - snprintf(c.name, tListLen(c.name), "%s.%s", tableName, pSchema->name); + char n[TSDB_COL_NAME_LEN + 1 + TSDB_TABLE_NAME_LEN] = {0}; + snprintf(n, tListLen(n), "%s.%s", tableName, pSchema->name); + + tstrncpy(c.name, n, tListLen(c.name)); } else { tstrncpy(c.name, pSchema->name, tListLen(c.name)); } From 4ff8072d3291f3082930517176e0074a58c313b8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Dec 2021 11:18:59 +0800 Subject: [PATCH 53/97] [td-11818] fix bug in show tables; --- source/client/test/clientTests.cpp | 54 ++++++++++++------------ source/dnode/vnode/impl/src/vnodeQuery.c | 4 +- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index f2c79bfd5a..97692d71b4 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -463,30 +463,30 @@ TEST(testCase, show_table_Test) { taos_close(pConn); } -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table t_2 using st1 tags(1) t_3 using st2 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1) t_3 using st2 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index a80828b6f7..01df929080 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -136,7 +136,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { metaCloseTbCursor(pCur); - int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; + int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; int32_t numOfTables = (int32_t) taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; @@ -150,7 +150,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { char* n = taosArrayGetP(pArray, i); STR_TO_VARSTR(p, n); - p += rowLen; + p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); } pFetchRsp->numOfRows = htonl(numOfTables); From ccc91cec3aa759b0550a58100d81225a684591ea Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Dec 2021 11:23:44 +0800 Subject: [PATCH 54/97] feature/qnode --- include/libs/qcom/query.h | 2 +- include/libs/scheduler/scheduler.h | 22 +- source/libs/catalog/inc/catalogInt.h | 26 +- source/libs/catalog/src/catalog.c | 135 ++++-- source/libs/catalog/test/catalogTests.cpp | 442 +++++++++++++++++- source/libs/scheduler/inc/schedulerInt.h | 4 +- source/libs/scheduler/src/scheduler.c | 31 +- source/libs/scheduler/test/schedulerTests.cpp | 7 +- source/util/src/thash.c | 18 +- 9 files changed, 577 insertions(+), 110 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 4ea35f1d2c..4d5b1a8bd3 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -75,7 +75,7 @@ typedef struct STableMeta { } STableMeta; typedef struct SDBVgroupInfo { - int32_t lock; + SRWLatch lock; int32_t vgVersion; int8_t hashMethod; SHashObj *vgInfo; //key:vgId, value:SVgroupInfo diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index ddcfcab4db..d6cac976d4 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -50,23 +50,37 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; +typedef struct SQueryNodeAddr{ + int32_t nodeId; //vgId or qnodeId + int8_t inUse; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SQueryNodeAddr; + +typedef struct SQueryResult { + int32_t code; + uint64_t numOfRows; + int32_t msgSize; + char *msg; +} SQueryResult; + int32_t schedulerInit(SSchedulerCfg *cfg); /** * Process the query job, generated according to the query physical plan. * This is a synchronized API, and is also thread-safety. - * @param qnodeList Qnode address list, element is SEpAddr + * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows); +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. * This is a asynchronized API, and is also thread-safety. - * @param qnodeList Qnode address list, element is SEpAddr + * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob); int32_t scheduleFetchRows(void *pJob, void **data); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 31b5939463..f426139c14 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -77,27 +77,37 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); #define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0) #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 + #define CTG_LOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ - if ((*(_lock)) < 0) assert(0); \ + assert(atomic_load_32((_lock)) >= 0); \ + ctgDebug("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ - ctgDebug("CTG RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + ctgDebug("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ } else { \ - if ((*(_lock)) < 0) assert(0); \ + assert(atomic_load_32((_lock)) >= 0); \ + ctgDebug("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ - ctgDebug("CTG WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + ctgDebug("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ } \ } while (0) #define CTG_UNLOCK(type, _lock) do { \ if (CTG_READ == (type)) { \ - if ((*(_lock)) <= 0) assert(0); \ + assert(atomic_load_32((_lock)) > 0); \ + ctgDebug("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ - ctgDebug("CTG RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + ctgDebug("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } else { \ - if ((*(_lock)) <= 0) assert(0); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ctgDebug("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ - ctgDebug("CTG WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + ctgDebug("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } \ } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b632ac772c..d881f226e4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -23,21 +23,32 @@ SCatalogMgmt ctgMgmt = {0}; int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, bool *inCache) { if (NULL == pCatalog->dbCache.cache) { *inCache = false; + ctgWarn("no db cache"); return TSDB_CODE_SUCCESS; } - SDBVgroupInfo *info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + SDBVgroupInfo *info = NULL; - if (NULL == info) { - *inCache = false; - return TSDB_CODE_SUCCESS; - } + while (true) { + info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - CTG_LOCK(CTG_READ, &info->lock); - if (NULL == info->vgInfo) { - CTG_UNLOCK(CTG_READ, &info->lock); - *inCache = false; - return TSDB_CODE_SUCCESS; + if (NULL == info) { + *inCache = false; + assert(0); + ctgWarn("no db cache, dbName:%s", dbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &info->lock); + if (NULL == info->vgInfo) { + CTG_UNLOCK(CTG_READ, &info->lock); + taosHashRelease(pCatalog->dbCache.cache, info); + ctgWarn("db cache vgInfo is NULL, dbName:%s", dbName); + + continue; + } + + break; } *dbInfo = info; @@ -271,8 +282,6 @@ _return: int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; - CTG_LOCK(CTG_READ, &dbInfo->lock); - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -311,8 +320,6 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const SName *pTableName *pVgroup = *vgInfo; _return: - - CTG_UNLOCK(CTG_READ, &dbInfo->lock); CTG_RET(TSDB_CODE_SUCCESS); } @@ -422,6 +429,8 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm if (0 == forceUpdate) { CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + assert(inCache); + if (inCache) { return TSDB_CODE_SUCCESS; } @@ -434,11 +443,47 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm input.db[sizeof(input.db) - 1] = 0; input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); + while (true) { + CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup)); + CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + + if (!inCache) { + ctgWarn("get db vgroup from cache failed, db:%s", dbName); + continue; + } + + break; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + SDBVgroupInfo *oldInfo = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (oldInfo) { + CTG_LOCK(CTG_WRITE, &oldInfo->lock); + if (dbInfo->vgVersion <= oldInfo->vgVersion) { + ctgInfo("dbName:%s vg will not update, vgVersion:%d , current:%d", dbName, dbInfo->vgVersion, oldInfo->vgVersion); + CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); + taosHashRelease(pCatalog->dbCache.cache, oldInfo); + + return TSDB_CODE_SUCCESS; + } + + if (oldInfo->vgInfo) { + ctgInfo("dbName:%s vg will be cleanup", dbName); + taosHashCleanup(oldInfo->vgInfo); + oldInfo->vgInfo = NULL; + } + + CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); + + taosHashRelease(pCatalog->dbCache.cache, oldInfo); + } return TSDB_CODE_SUCCESS; } @@ -581,55 +626,57 @@ _return: int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { + int32_t code = 0; + if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == dbInfo->vgInfo || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgInfo) <= 0) { + ctgError("invalid db vg, dbName:%s", dbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } if (dbInfo->vgVersion < 0) { - if (pCatalog->dbCache.cache) { - SDBVgroupInfo *oldInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo) { - CTG_LOCK(CTG_WRITE, &oldInfo->lock); - if (oldInfo->vgInfo) { - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; - } - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); + ctgWarn("invalid db vgVersion:%d, dbName:%s", dbInfo->vgVersion, dbName); - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - } + if (pCatalog->dbCache.cache) { + CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); + + CTG_ERR_JRET(taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName))); } ctgWarn("remove db [%s] from cache", dbName); - return TSDB_CODE_SUCCESS; + goto _return; } if (NULL == pCatalog->dbCache.cache) { pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->dbCache.cache) { ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } else { - SDBVgroupInfo *oldInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo) { - CTG_LOCK(CTG_WRITE, &oldInfo->lock); - if (oldInfo->vgInfo) { - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; - } - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); - - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - } + CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); } if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) { ctgError("push to vgroup hash cache failed"); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - return TSDB_CODE_SUCCESS; + ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, dbInfo->vgVersion); + + dbInfo->vgInfo = NULL; + +_return: + + if (dbInfo && dbInfo->vgInfo) { + taosHashCleanup(dbInfo->vgInfo); + dbInfo->vgInfo = NULL; + } + + CTG_RET(code); } int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 0ad00046cd..5bacbbd7e7 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -36,11 +36,19 @@ namespace { +extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist); +extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); + void ctgTestSetPrepareTableMeta(); void ctgTestSetPrepareCTableMeta(); void ctgTestSetPrepareSTableMeta(); +bool ctgTestStop = false; +bool ctgTestEnableSleep = false; +bool ctgTestDeadLoop = true; +int32_t ctgTestCurrentVgVersion = 0; +int32_t ctgTestVgVersion = 1; int32_t ctgTestVgNum = 10; int32_t ctgTestColNum = 2; int32_t ctgTestTagNum = 1; @@ -89,6 +97,113 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { ASSERT_EQ(rpcRsp.code, 0); } +void ctgTestInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + ctgDebugFlag = 159; + tsAsyncLog = 0; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } + +} + +int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { + return ((vgVersion % 2) == 0) ? ctgTestVgNum - 2 : ctgTestVgNum; +} + +void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { + SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(cn.dbname, "db1"); + strcpy(cn.tname, ctgTestCTablename); + + SName sn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(sn.dbname, "db1"); + strcpy(sn.tname, ctgTestSTablename); + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&cn, tbFullName); + + output->metaNum = 2; + + strcpy(output->ctbFname, tbFullName); + + tNameExtractFullName(&cn, tbFullName); + strcpy(output->tbFname, tbFullName); + + output->ctbMeta.vgId = 9; + output->ctbMeta.tableType = TSDB_CHILD_TABLE; + output->ctbMeta.uid = 3; + output->ctbMeta.suid = 2; + + output->tbMeta = (STableMeta *)calloc(1, sizeof(STableMeta) + sizeof(SSchema) * (ctgTestColNum + ctgTestColNum)); + output->tbMeta->vgId = 9; + output->tbMeta->tableType = TSDB_SUPER_TABLE; + output->tbMeta->uid = 2; + output->tbMeta->suid = 2; + + output->tbMeta->tableInfo.numOfColumns = ctgTestColNum; + output->tbMeta->tableInfo.numOfTags = ctgTestTagNum; + + output->tbMeta->sversion = ctgTestSVersion; + output->tbMeta->tversion = ctgTestTVersion; + + SSchema *s = NULL; + s = &output->tbMeta->schema[0]; + s->type = TSDB_DATA_TYPE_TIMESTAMP; + s->colId = 1; + s->bytes = 8; + strcpy(s->name, "ts"); + + s = &output->tbMeta->schema[1]; + s->type = TSDB_DATA_TYPE_INT; + s->colId = 2; + s->bytes = 4; + strcpy(s->name, "col1s"); + + s = &output->tbMeta->schema[2]; + s->type = TSDB_DATA_TYPE_BINARY; + s->colId = 3; + s->bytes = 12; + strcpy(s->name, "tag1s"); + +} + +void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { + static int32_t vgVersion = ctgTestVgVersion + 1; + int32_t vgNum = 0; + SVgroupInfo vgInfo = {0}; + + dbVgroup->vgVersion = vgVersion++; + + ctgTestCurrentVgVersion = dbVgroup->vgVersion; + + dbVgroup->hashMethod = 0; + dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + + vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion); + uint32_t hashUnit = UINT32_MAX / vgNum; + + for (int32_t i = 0; i < vgNum; ++i) { + vgInfo.vgId = i + 1; + vgInfo.hashBegin = i * hashUnit; + vgInfo.hashEnd = hashUnit * (i + 1) - 1; + vgInfo.numOfEps = i % TSDB_MAX_REPLICA + 1; + vgInfo.inUse = i % vgInfo.numOfEps; + for (int32_t n = 0; n < vgInfo.numOfEps; ++n) { + SEpAddrMsg *addr = &vgInfo.epAddr[n]; + strcpy(addr->fqdn, "a0"); + addr->port = htons(n + 22); + } + + taosHashPut(dbVgroup->vgInfo, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); + } +} + void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SUseDbRsp *rspMsg = NULL; //todo @@ -97,7 +212,8 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (SUseDbRsp *)pRsp->pCont; strcpy(rspMsg->db, ctgTestDbname); - rspMsg->vgVersion = htonl(1); + rspMsg->vgVersion = htonl(ctgTestVgVersion); + ctgTestCurrentVgVersion = ctgTestVgVersion; rspMsg->vgNum = htonl(ctgTestVgNum); rspMsg->hashMethod = 0; @@ -148,13 +264,13 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM SSchema *s = NULL; s = &rspMsg->pSchema[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(0); + s->colId = htonl(1); s->bytes = htonl(8); strcpy(s->name, "ts"); s = &rspMsg->pSchema[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(1); + s->colId = htonl(2); s->bytes = htonl(4); strcpy(s->name, "col1"); @@ -185,19 +301,19 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc SSchema *s = NULL; s = &rspMsg->pSchema[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(0); + s->colId = htonl(1); s->bytes = htonl(8); strcpy(s->name, "ts"); s = &rspMsg->pSchema[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(1); + s->colId = htonl(2); s->bytes = htonl(4); strcpy(s->name, "col1s"); s = &rspMsg->pSchema[2]; s->type = TSDB_DATA_TYPE_BINARY; - s->colId = htonl(2); + s->colId = htonl(3); s->bytes = htonl(12); strcpy(s->name, "tag1s"); @@ -229,19 +345,19 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc SSchema *s = NULL; s = &rspMsg->pSchema[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(0); + s->colId = htonl(1); s->bytes = htonl(8); strcpy(s->name, "ts"); s = &rspMsg->pSchema[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(1); + s->colId = htonl(2); s->bytes = htonl(4); strcpy(s->name, "col1s"); s = &rspMsg->pSchema[2]; s->type = TSDB_DATA_TYPE_BINARY; - s->colId = htonl(2); + s->colId = htonl(3); s->bytes = htonl(12); strcpy(s->name, "tag1s"); @@ -371,6 +487,117 @@ void ctgTestSetPrepareDbVgroupsAndSuperMeta() { } +void *ctgTestGetDbVgroupThread(void *param) { + struct SCatalog* pCtg = (struct SCatalog*)param; + int32_t code = 0; + void *mockPointer = (void *)0x1; + SArray *vgList = NULL; + int32_t n = 0; + + while (!ctgTestStop) { + code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + if (code) { + assert(0); + } + + if (vgList) { + taosArrayDestroy(vgList); + } + + if (ctgTestEnableSleep) { + usleep(rand()%5); + } + if (++n % 50000 == 0) { + printf("Get:%d\n", n); + } + } + + return NULL; +} + +void *ctgTestSetDbVgroupThread(void *param) { + struct SCatalog* pCtg = (struct SCatalog*)param; + int32_t code = 0; + SDBVgroupInfo dbVgroup = {0}; + int32_t n = 0; + + while (!ctgTestStop) { + ctgTestBuildDBVgroup(&dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + if (code) { + assert(0); + } + + if (ctgTestEnableSleep) { + usleep(rand()%5); + } + if (++n % 50000 == 0) { + printf("Set:%d\n", n); + } + } + + return NULL; + +} + +void *ctgTestGetCtableMetaThread(void *param) { + struct SCatalog* pCtg = (struct SCatalog*)param; + int32_t code = 0; + int32_t n = 0; + STableMeta* tbMeta = NULL; + int32_t exist = 0; + + SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(cn.dbname, "db1"); + strcpy(cn.tname, ctgTestCTablename); + + while (!ctgTestStop) { + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist); + if (code || 0 == exist) { + assert(0); + } + + if (ctgTestEnableSleep) { + usleep(rand()%5); + } + + if (++n % 50000 == 0) { + printf("Get:%d\n", n); + } + } + + return NULL; +} + +void *ctgTestSetCtableMetaThread(void *param) { + struct SCatalog* pCtg = (struct SCatalog*)param; + int32_t code = 0; + SDBVgroupInfo dbVgroup = {0}; + int32_t n = 0; + STableMetaOutput output = {0}; + + ctgTestBuildCTableMetaOutput(&output); + + while (!ctgTestStop) { + code = ctgUpdateTableMetaCache(pCtg, &output); + if (code) { + assert(0); + } + + if (ctgTestEnableSleep) { + usleep(rand()%5); + } + if (++n % 50000 == 0) { + printf("Set:%d\n", n); + } + } + + return NULL; + +} + +#if 0 + TEST(tableMeta, normalTable) { struct SCatalog* pCtg = NULL; void *mockPointer = (void *)0x1; @@ -388,7 +615,7 @@ TEST(tableMeta, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = T_NAME_TABLE, .acctId = 1}; + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -436,11 +663,13 @@ TEST(tableMeta, childTableCase) { initQueryModuleMsgHandle(); //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); - int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg); + code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = T_NAME_TABLE, .acctId = 1}; + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -494,11 +723,14 @@ TEST(tableMeta, superTableCase) { initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - SName n = {.type = T_NAME_TABLE, .acctId = 1}; + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -558,12 +790,15 @@ TEST(tableDistVgroup, normalTable) { initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - - int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - SName n = {.type = T_NAME_TABLE, .acctId = 1}; + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -595,7 +830,7 @@ TEST(tableDistVgroup, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = T_NAME_TABLE, .acctId = 1}; + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -620,11 +855,14 @@ TEST(tableDistVgroup, superTableCase) { initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - int32_t code = catalogGetHandle(ctgTestClusterId, &pCtg); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - SName n = {.type = T_NAME_TABLE, .acctId = 1}; + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -645,6 +883,164 @@ TEST(tableDistVgroup, superTableCase) { catalogDestroy(); } +TEST(dbVgroup, getSetDbVgroupCase) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; + + ctgTestSetPrepareDbVgroupsAndNormalMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); + ASSERT_EQ(code, 0); + ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 8); + ASSERT_EQ(vgInfo.numOfEps, 3); + + code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + ASSERT_EQ(code, 0); + ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); + pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); + ASSERT_EQ(pvgInfo->vgId, 8); + ASSERT_EQ(pvgInfo->numOfEps, 3); + taosArrayDestroy(vgList); + + ctgTestBuildDBVgroup(&dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + ASSERT_EQ(code, 0); + + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); + ASSERT_EQ(code, 0); + ASSERT_EQ(vgInfo.vgId, 7); + ASSERT_EQ(vgInfo.numOfEps, 2); + + code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); + ASSERT_EQ(code, 0); + ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); + pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); + ASSERT_EQ(pvgInfo->vgId, 8); + ASSERT_EQ(pvgInfo->numOfEps, 3); + taosArrayDestroy(vgList); + + catalogDestroy(); +} + +#endif + +TEST(multiThread, getSetDbVgroupCase) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; + + ctgTestInitLogFile(); + + ctgTestSetPrepareDbVgroups(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t thread1, thread2; + pthread_create(&(thread1), &thattr, ctgTestSetDbVgroupThread, pCtg); + + sleep(1); + pthread_create(&(thread1), &thattr, ctgTestGetDbVgroupThread, pCtg); + + while (true) { + if (ctgTestDeadLoop) { + sleep(1); + } else { + sleep(600); + break; + } + } + + ctgTestStop = true; + sleep(1); + + catalogDestroy(); +} + +TEST(multiThread, ctableMeta) { + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; + + ctgTestSetPrepareDbVgroupsAndChildMeta(); + + initQueryModuleMsgHandle(); + + //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(ctgTestClusterId, &pCtg); + ASSERT_EQ(code, 0); + + SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + strcpy(n.dbname, "db1"); + strcpy(n.tname, ctgTestTablename); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + + pthread_t thread1, thread2; + pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); + pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); + + while (true) { + if (ctgTestDeadLoop) { + sleep(1); + } else { + sleep(600); + break; + } + } + + ctgTestStop = true; + sleep(1); + + catalogDestroy(); +} int main(int argc, char** argv) { diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index c327e4cfea..2770f7e21a 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -89,12 +89,12 @@ typedef struct SSchJob { SEpSet dataSrcEps; SEpAddr resEp; void *transport; - SArray *qnodeList; + SArray *nodeList; // qnode/vnode list, element is SQueryNodeAddr tsem_t rspSem; int32_t userFetch; int32_t remoteFetch; - SSchTask *fetchTask; + SSchTask *fetchTask; int32_t errCode; void *res; int32_t resNumOfRows; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 7bd2205e43..429211bdb8 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -220,10 +220,10 @@ int32_t schSetTaskExecEpSet(SSchJob *job, SEpSet *epSet) { return TSDB_CODE_SUCCESS; } - int32_t qnodeNum = taosArrayGetSize(job->qnodeList); + int32_t nodeNum = taosArrayGetSize(job->nodeList); - for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) { - SEpAddr *addr = taosArrayGet(job->qnodeList, i); + for (int32_t i = 0; i < nodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) { + SEpAddr *addr = taosArrayGet(job->nodeList, i); strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn)); epSet->port[epSet->numOfEps] = addr->port; @@ -829,8 +829,8 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } -int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) { - if (qnodeList && taosArrayGetSize(qnodeList) <= 0) { +int32_t scheduleExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, bool syncSchedule) { + if (nodeList && taosArrayGetSize(nodeList) <= 0) { qInfo("qnodeList is empty"); } @@ -842,7 +842,7 @@ int32_t scheduleExecJobImpl(void *transport, SArray *qnodeList, SQueryDag* pDag, job->attr.syncSchedule = syncSchedule; job->transport = transport; - job->qnodeList = qnodeList; + job->nodeList = nodeList; SCH_ERR_JRET(schValidateAndBuildJob(pDag, job)); @@ -897,28 +897,27 @@ _return: SCH_RET(code); } -int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob, uint64_t *numOfRows) { - if (NULL == transport || /* NULL == qnodeList || */ NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == numOfRows) { +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes) { + if (NULL == transport || /* NULL == nodeList || */ NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - *numOfRows = 0; - - SCH_ERR_RET(scheduleExecJobImpl(transport, qnodeList, pDag, pJob, true)); + SCH_ERR_RET(scheduleExecJobImpl(transport, nodeList, pDag, pJob, true)); SSchJob *job = *(SSchJob **)pJob; - *numOfRows = job->resNumOfRows; - + pRes->code = job->errCode; + pRes->numOfRows = job->resNumOfRows; + return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { - if (NULL == transport || NULL == qnodeList ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { + if (NULL == transport || NULL == nodeList ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - return scheduleExecJobImpl(transport, qnodeList, pDag, pJob, false); + return scheduleExecJobImpl(transport, nodeList, pDag, pJob, false); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6163bc0c1a..b418ade172 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -321,10 +321,11 @@ TEST(insertTest, normalCase) { pthread_t thread1; pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); - - code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &numOfRows); + + SQueryResult res = {0}; + code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); ASSERT_EQ(code, 0); - ASSERT_EQ(numOfRows, 20); + ASSERT_EQ(res.numOfRows, 20); scheduleFreeJob(pInsertJob); } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 840a1ef390..2841f27da4 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -132,7 +132,7 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* } else { pNewNode->next = pNode; pe->num++; - atomic_add_fetch_64(&pHashObj->size, 1); + atomic_add_fetch_32(&pHashObj->size, 1); } return pNewNode; @@ -209,7 +209,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { if (!pHashObj) { return 0; } - return (int32_t)atomic_load_64(&pHashObj->size); + return (int32_t)atomic_load_32(&pHashObj->size); } static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { @@ -273,7 +273,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize __rd_unlock(&pHashObj->lock, pHashObj->type); - atomic_add_fetch_64(&pHashObj->size, 1); + atomic_add_fetch_32(&pHashObj->size, 1); return 0; } else { @@ -405,7 +405,7 @@ void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v } if (acquire) { - pNode->count++; + atomic_add_fetch_16(&pNode->count, 1); } data = GET_HASH_NODE_DATA(pNode); @@ -482,7 +482,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi // if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); pe->num--; - atomic_sub_fetch_64(&pHashObj->size, 1); + atomic_sub_fetch_32(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pNode); } } @@ -520,7 +520,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi while((pNode = pEntry->next) != NULL) { if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { pEntry->num -= 1; - atomic_sub_fetch_64(&pHashObj->size, 1); + atomic_sub_fetch_32(&pHashObj->size, 1); pEntry->next = pNode->next; @@ -546,7 +546,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { pNode->next = pNext->next; pEntry->num -= 1; - atomic_sub_fetch_64(&pHashObj->size, 1); + atomic_sub_fetch_32(&pHashObj->size, 1); if (pEntry->num == 0) { assert(pEntry->next == NULL); @@ -600,7 +600,7 @@ void taosHashClear(SHashObj *pHashObj) { pEntry->next = NULL; } - pHashObj->size = 0; + atomic_store_32(&pHashObj->size, 0); __wr_unlock(&pHashObj->lock, pHashObj->type); } @@ -847,7 +847,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } pe->num--; - atomic_sub_fetch_64(&pHashObj->size, 1); + atomic_sub_fetch_32(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pOld); } } else { From 8e9183a04a4b64d489ddc51d95cec7ef912f5120 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 03:39:51 +0000 Subject: [PATCH 55/97] more --- include/common/tmsg.h | 5 +++++ include/dnode/vnode/meta/meta.h | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 17a4d4ad2c..6bb5406bba 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1271,6 +1271,11 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; +typedef struct { + uint64_t ver; // use a general definition + SArray* pArray; +} SVCreateTbBatchReq; + int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq); diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index b75e478add..d587107270 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -59,9 +59,9 @@ STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); -SMTbCursor * metaOpenTbCursor(SMeta *pMeta); -void metaCloseTbCursor(SMTbCursor *pTbCur); -char *metaTbCursorNext(SMTbCursor *pTbCur); +SMTbCursor *metaOpenTbCursor(SMeta *pMeta); +void metaCloseTbCursor(SMTbCursor *pTbCur); +char * metaTbCursorNext(SMTbCursor *pTbCur); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); From dc8bad19df35ad3a3300deb5f03a36f67940c42e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 03:45:44 +0000 Subject: [PATCH 56/97] more --- include/common/tmsg.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6bb5406bba..80dec89611 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -23,6 +23,7 @@ extern "C" { #include "encode.h" #include "taosdef.h" #include "taoserror.h" +#include "tarray.h" #include "tcoding.h" #include "tdataformat.h" #include "tlist.h" From b4203385514d5ccabf281ad14c4b07ab7b0a14be Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Dec 2021 13:34:38 +0800 Subject: [PATCH 57/97] feature/qnode --- include/common/tmsg.h | 7 +++++-- source/dnode/mgmt/impl/src/dndTransport.c | 4 ++-- source/dnode/vnode/impl/src/vnodeQuery.c | 18 ++++++++++++++++-- source/libs/catalog/src/catalog.c | 14 +++++--------- source/libs/qcom/src/querymsg.c | 2 +- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 17a4d4ad2c..0b7ba24c6b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -774,8 +774,8 @@ typedef struct { } SAuthVnodeMsg; typedef struct { - int32_t vgId; - char tableFname[TSDB_TABLE_FNAME_LEN]; + SMsgHead header; + char tableFname[TSDB_TABLE_FNAME_LEN]; } STableInfoMsg; typedef struct { @@ -1059,6 +1059,7 @@ typedef struct { } SUpdateTagValRsp; typedef struct SSubQueryMsg { + SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; @@ -1067,6 +1068,7 @@ typedef struct SSubQueryMsg { } SSubQueryMsg; typedef struct SResReadyMsg { + SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; @@ -1077,6 +1079,7 @@ typedef struct SResReadyRsp { } SResReadyRsp; typedef struct SResFetchMsg { + SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index e5be16937b..dc5248c8e0 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -121,8 +121,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLES_META)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONSUME)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_QUERY)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_CONNECT)] = dndProcessVnodeWriteMsg; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 01df929080..daa1e964de 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -17,6 +17,7 @@ #include "vnodeDef.h" static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); +static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } @@ -43,6 +44,8 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TDMT_VND_SHOW_TABLES_FETCH: return vnodeGetTableList(pVnode, pMsg); // return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); + case TDMT_VND_TABLE_META: + return vnodeGetTableMeta(pVnode, pMsg, pRsp); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -88,7 +91,8 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pTagSchema = NULL; } - pTbMetaMsg = (STableMetaMsg *)calloc(1, sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols)); + int msgLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols); + pTbMetaMsg = (STableMetaMsg *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { return -1; } @@ -115,6 +119,16 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pSch->bytes = htonl(pSch->bytes); } + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pTbMetaMsg, + .contLen = msgLen, + .code = 0, + }; + + rpcSendResponse(&rpcMsg); + return 0; } @@ -166,4 +180,4 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { rpcSendResponse(&rpcMsg); return 0; -} \ No newline at end of file +} diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b8fa779e70..68ff1b8557 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -34,7 +34,6 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S if (NULL == info) { *inCache = false; - assert(0); ctgWarn("no db cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } @@ -190,14 +189,13 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE } -int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } char tbFullName[TSDB_TABLE_FNAME_LEN]; - - snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); + tNameExtractFullName(pTableName, tbFullName); SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName}; char *msg = NULL; @@ -429,8 +427,6 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm if (0 == forceUpdate) { CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); - assert(inCache); - if (inCache) { return TSDB_CODE_SUCCESS; } @@ -694,9 +690,9 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSe STableMetaOutput output = {0}; - //CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output)); + CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pTableName, &vgroupInfo, &output)); - CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output)); + //CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output)); CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output)); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 27ca406fc4..117297b9ff 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -40,7 +40,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 STableInfoMsg *bMsg = (STableInfoMsg *)*msg; - bMsg->vgId = bInput->vgId; + bMsg->header.vgId = htonl(bInput->vgId); strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; From 76e393d4a8f354eef5e4ce6feb5ea300278ff856 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 05:41:58 +0000 Subject: [PATCH 58/97] more --- include/common/tmsg.h | 18 +++++----- include/util/freelist.h | 59 +++++++++++++++++++++++++++++++ source/util/test/CMakeLists.txt | 8 +++++ source/util/test/freelistTest.cpp | 16 +++++++++ 4 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 include/util/freelist.h create mode 100644 source/util/test/freelistTest.cpp diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 80dec89611..bf7a6ea11b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -355,9 +355,9 @@ typedef struct SEpSet { } SEpSet; static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { - if(buf == NULL) return sizeof(SEpSet); + if (buf == NULL) return sizeof(SEpSet); memcpy(buf, pEp, sizeof(SEpSet)); - //TODO: endian conversion + // TODO: endian conversion return sizeof(SEpSet); } @@ -1119,10 +1119,10 @@ typedef struct STaskDropRsp { } STaskDropRsp; typedef struct { - int8_t igExists; - char* name; - char* physicalPlan; - char* logicalPlan; + int8_t igExists; + char* name; + char* physicalPlan; + char* logicalPlan; } SCMCreateTopicReq; static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) { @@ -1158,8 +1158,8 @@ static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopi } typedef struct { - char* topicName; - char* consumerGroup; + char* topicName; + char* consumerGroup; int64_t consumerId; } SCMSubscribeReq; @@ -1180,7 +1180,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq typedef struct { int32_t vgId; - SEpSet pEpSet; + SEpSet pEpSet; } SCMSubscribeRsp; static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { diff --git a/include/util/freelist.h b/include/util/freelist.h new file mode 100644 index 0000000000..497a6d58c3 --- /dev/null +++ b/include/util/freelist.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_FREELIST_H_ +#define _TD_UTIL_FREELIST_H_ + +#include "os.h" +#include "tlist.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct SFreeListNode { + TD_SLIST_NODE(SFreeListNode); + char payload[]; +}; + +typedef TD_SLIST(SFreeListNode) SFreeList; + +#define TFL_MALLOC(SIZE, LIST) \ + ({ \ + void *ptr = malloc((SIZE) + sizeof(struct SFreeListNode)); \ + if (ptr) { \ + TD_SLIST_PUSH((LIST), (struct SFreeListNode *)ptr); \ + ptr = ((struct SFreeListNode *)ptr)->payload; \ + } \ + ptr; \ + }) + +#define tFreeListInit(pFL) TD_SLIST_INIT(pFL) + +static FORCE_INLINE void tFreeListClear(SFreeList *pFL) { + struct SFreeListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(pFL); + if (pNode == NULL) break; + TD_SLIST_POP(pFL); + free(pNode); + } +} + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_FREELIST_H_*/ \ No newline at end of file diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 79aaa1beb0..bfc3906f79 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -33,4 +33,12 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +# freelistTest +add_executable(freelistTest "") +target_sources(freelistTest + PRIVATE + "freelistTest.cpp" +) +target_link_libraries(freelistTest os util gtest gtest_main) + diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp new file mode 100644 index 0000000000..7a4e8be5b7 --- /dev/null +++ b/source/util/test/freelistTest.cpp @@ -0,0 +1,16 @@ +#include "gtest/gtest.h" + +#include "freelist.h" + +TEST(TD_UTIL_FREELIST_TEST, simple_test) { + SFreeList fl; + + tFreeListInit(&fl); + + for (size_t i = 0; i < 1000; i++) { + void *ptr = TFL_MALLOC(1024, &fl); + GTEST_ASSERT_NE(ptr, nullptr); + } + + tFreeListClear(&fl); +} \ No newline at end of file From 5ee6cbd4e383c8b40871c0a5731d56757087140f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 06:02:48 +0000 Subject: [PATCH 59/97] more --- source/dnode/vnode/impl/src/vnodeQuery.c | 50 ++++++++++++------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index daa1e964de..d869c245d9 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -17,7 +17,7 @@ #include "vnodeDef.h" static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); -static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } @@ -43,7 +43,7 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return qWorkerProcessShowMsg(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_SHOW_TABLES_FETCH: return vnodeGetTableList(pVnode, pMsg); -// return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); + // return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg, pRsp); default: @@ -62,16 +62,17 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SSchemaWrapper *pSW; STableMetaMsg * pTbMetaMsg; SSchema * pTagSchema; + SRpcMsg rpcMsg; pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); if (pTbCfg == NULL) { - return -1; + goto _exit; } if (pTbCfg->type == META_CHILD_TABLE) { pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); if (pStbCfg == NULL) { - return -1; + goto _exit; } pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true); @@ -94,7 +95,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int msgLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols); pTbMetaMsg = (STableMetaMsg *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { - return -1; + goto _exit; } strcpy(pTbMetaMsg->tbFname, pTbCfg->name); @@ -119,13 +120,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pSch->bytes = htonl(pSch->bytes); } - SRpcMsg rpcMsg = { - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, - .pCont = pTbMetaMsg, - .contLen = msgLen, - .code = 0, - }; +_exit: + + rpcMsg.handle = pMsg->handle; + rpcMsg.ahandle = pMsg->ahandle; + rpcMsg.pCont = pTbMetaMsg; + rpcMsg.contLen = msgLen; + rpcMsg.code = 0; rpcSendResponse(&rpcMsg); @@ -138,10 +139,10 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { * @param pRsp */ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { - SMTbCursor* pCur = metaOpenTbCursor(pVnode->pMeta); - SArray* pArray = taosArrayInit(10, POINTER_BYTES); + SMTbCursor *pCur = metaOpenTbCursor(pVnode->pMeta); + SArray * pArray = taosArrayInit(10, POINTER_BYTES); - char* name = NULL; + char * name = NULL; int32_t totalLen = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { taosArrayPush(pArray, &name); @@ -150,18 +151,19 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { metaCloseTbCursor(pCur); - int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - int32_t numOfTables = (int32_t) taosArrayGetSize(pArray); + int32_t rowLen = + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; + int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; -// SVShowTablesFetchReq *pFetchReq = pMsg->pCont; + // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp) + payloadLen); memset(pFetchRsp, 0, sizeof(struct SVShowTablesFetchRsp) + payloadLen); - char* p = pFetchRsp->data; - for(int32_t i = 0; i < numOfTables; ++i) { - char* n = taosArrayGetP(pArray, i); + char *p = pFetchRsp->data; + for (int32_t i = 0; i < numOfTables; ++i) { + char *n = taosArrayGetP(pArray, i); STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); @@ -171,11 +173,11 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { pFetchRsp->precision = 0; SRpcMsg rpcMsg = { - .handle = pMsg->handle, + .handle = pMsg->handle, .ahandle = pMsg->ahandle, - .pCont = pFetchRsp, + .pCont = pFetchRsp, .contLen = sizeof(SVShowTablesFetchRsp) + payloadLen, - .code = 0, + .code = 0, }; rpcSendResponse(&rpcMsg); From 953ef456ef6def74e884b6653110d8a6f1c07639 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Dec 2021 14:26:05 +0800 Subject: [PATCH 60/97] feature/qnode --- include/common/tmsg.h | 2 ++ source/libs/catalog/src/catalog.c | 29 ++++++++++------------- source/libs/catalog/test/catalogTests.cpp | 11 +++++++-- source/libs/qcom/src/querymsg.c | 5 ++++ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5cf027591f..483337f43b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -174,6 +174,7 @@ typedef enum _mgmt_table { typedef struct SBuildTableMetaInput { int32_t vgId; + char* dbName; char* tableFullName; } SBuildTableMetaInput; @@ -776,6 +777,7 @@ typedef struct { typedef struct { SMsgHead header; + char dbFname[TSDB_DB_FNAME_LEN]; char tableFname[TSDB_TABLE_FNAME_LEN]; } STableInfoMsg; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 68ff1b8557..236264873e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -161,7 +161,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE char tbFullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFullName); - SBuildTableMetaInput bInput = {.vgId = 0, .tableFullName = tbFullName}; + SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; @@ -194,10 +194,10 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbFullName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFullName); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .tableFullName = tbFullName}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = pTableName->tname}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; @@ -355,19 +355,19 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out if (output->metaNum != 1 && output->metaNum != 2) { ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == output->tbMeta) { ctgError("no valid table meta got from meta rsp"); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == pCatalog->tableCache.cache) { pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -375,19 +375,19 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.stableCache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } if (output->metaNum == 2) { if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { ctgError("push ctable[%s] to table cache failed", output->ctbFname); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } if (TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } } @@ -398,26 +398,23 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); ctgError("push table[%s] to table cache failed", output->tbFname); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname)); if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES) != 0) { CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); } else { if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { ctgError("push table[%s] to table cache failed", output->tbFname); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } -_return: - tfree(output->tbMeta); - CTG_RET(code); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 5979d3a147..1d8a48dfcb 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -557,6 +557,8 @@ void *ctgTestGetCtableMetaThread(void *param) { assert(0); } + tfree(tbMeta); + if (ctgTestEnableSleep) { usleep(rand()%5); } @@ -592,6 +594,8 @@ void *ctgTestSetCtableMetaThread(void *param) { } } + tfree(output.tbMeta); + return NULL; } @@ -944,7 +948,6 @@ TEST(dbVgroup, getSetDbVgroupCase) { catalogDestroy(); } -#endif TEST(multiThread, getSetDbVgroupCase) { struct SCatalog* pCtg = NULL; @@ -996,6 +999,9 @@ TEST(multiThread, getSetDbVgroupCase) { catalogDestroy(); } +#endif + + TEST(multiThread, ctableMeta) { struct SCatalog* pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1024,8 +1030,9 @@ TEST(multiThread, ctableMeta) { pthread_attr_init(&thattr); pthread_t thread1, thread2; - pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); pthread_create(&(thread1), &thattr, ctgTestSetCtableMetaThread, pCtg); + sleep(1); + pthread_create(&(thread1), &thattr, ctgTestGetCtableMetaThread, pCtg); while (true) { if (ctgTestDeadLoop) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 117297b9ff..1e27749a1a 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -42,6 +42,11 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 bMsg->header.vgId = htonl(bInput->vgId); + if (bInput->dbName) { + strncpy(bMsg->dbFname, bInput->dbName, sizeof(bMsg->dbFname)); + bMsg->dbFname[sizeof(bMsg->dbFname) - 1] = 0; + } + strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; From 86c2d4b42e6452c8496918408961aced4d4e6215 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Dec 2021 14:43:04 +0800 Subject: [PATCH 61/97] [td-11818] support create table in batch. --- source/client/test/clientTests.cpp | 60 ++++++++++++------------- source/libs/parser/src/dCDAstProcess.c | 62 +++++++++++++++----------- 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 97692d71b4..7a87acfc36 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -435,42 +435,14 @@ TEST(testCase, connect_Test) { // taos_close(pConn); //} -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -//TEST(testCase, create_multiple_tables) { +//TEST(testCase, show_table_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // // TAOS_RES* pRes = taos_query(pConn, "use abc1"); // taos_free_result(pRes); // -// pRes = taos_query(pConn, "create table t_2 using st1 tags(1) t_3 using st2 tags(2)"); +// pRes = taos_query(pConn, "show tables"); // if (taos_errno(pRes) != 0) { // printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); @@ -490,3 +462,31 @@ TEST(testCase, show_table_Test) { // taos_free_result(pRes); // taos_close(pConn); //} + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1) t_3 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index fd99cb6f66..024870014b 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -321,6 +321,11 @@ int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } +typedef struct SVgroupTablesBatch { + SVCreateTbBatchReq req; + SVgroupInfo info; +} SVgroupTablesBatch; + int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len, SEpSet* pEpSet) { const char* msg1 = "invalid table name"; @@ -330,17 +335,14 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + // super table name, create table by using dst - int32_t numOfTables = (int32_t)taosArrayGetSize(pCreateTable->childTableInfo); + size_t numOfTables = taosArrayGetSize(pCreateTable->childTableInfo); for (int32_t j = 0; j < numOfTables; ++j) { SCreatedTableInfo* pCreateTableInfo = taosArrayGet(pCreateTable->childTableInfo, j); SToken* pSTableNameToken = &pCreateTableInfo->stbName; - - char buf[TSDB_TABLE_FNAME_LEN]; - SToken sTblToken; - sTblToken.z = buf; - int32_t code = parserValidateNameToken(pSTableNameToken); if (code != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); @@ -460,12 +462,13 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p for (int32_t i = 0; i < numOfInputTag; ++i) { SSchema* pSchema = &pTagSchema[i]; - SToken* pItem = taosArrayGet(pValList, i); - - char tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0}; - SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema}; char* endPtr = NULL; + char tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0}; + + SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema}; + + SToken* pItem = taosArrayGet(pValList, i); code = parseValueToken(&endPtr, pItem, pSchema, tinfo.precision, tmpTokenBuf, KvRowAppend, ¶m, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { @@ -478,7 +481,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder); if (row == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_QRY_OUT_OF_MEMORY; } tdSortKVRowByColIdx(row); @@ -489,31 +492,36 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p return code; } + SVgroupInfo info = {0}; + catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info); + struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; req.name = strdup(tNameGetTableName(&tableName)); req.ctbCfg.suid = pSuperTableMeta->suid; req.ctbCfg.pTag = row; - int32_t serLen = sizeof(SMsgHead) + tSerializeSVCreateTbReq(NULL, &req); - char* buf1 = calloc(1, serLen); - *pOutput = buf1; - buf1 += sizeof(SMsgHead); - tSerializeSVCreateTbReq((void*)&buf1, &req); - *len = serLen; +// pEpSet->inUse = info.inUse; +// pEpSet->numOfEps = info.numOfEps; +// for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { +// pEpSet->port[i] = info.epAddr[i].port; +// tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); +// } +// ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId); +// ((SMsgHead*)(*pOutput))->contLen = htonl(serLen); + SVgroupTablesBatch *pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId)); + if (pTableBatch == NULL) { + SVgroupTablesBatch tBatch = {0}; + tBatch.info = info; - SVgroupInfo info = {0}; - catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info); + tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); + taosArrayPush(tBatch.req.pArray, &req); - pEpSet->inUse = info.inUse; - pEpSet->numOfEps = info.numOfEps; - for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - pEpSet->port[i] = info.epAddr[i].port; - tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + taosHashPut(pVgroupHashmap, &info.vgId, sizeof(info.vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + assert(info.vgId == pTableBatch->info.vgId); + taosArrayPush(pTableBatch->req.pArray, &req); } - - ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId); - ((SMsgHead*)(*pOutput))->contLen = htonl(serLen); } return TSDB_CODE_SUCCESS; From 36437b65cecaa32a697fafccce3402606e5a8e0e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Dec 2021 14:43:13 +0800 Subject: [PATCH 62/97] feature/qnode --- include/common/tmsg.h | 1 + source/dnode/vnode/impl/src/vnodeQuery.c | 19 ++++++++++++------- source/libs/qcom/src/querymsg.c | 17 +++++++++++++---- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index cf3712b444..39f50e0496 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -812,6 +812,7 @@ typedef struct { typedef struct { char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name char stbFname[TSDB_TABLE_FNAME_LEN]; + char dbFname[TSDB_DB_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; int8_t precision; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index d869c245d9..cbc4d75e8b 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -60,9 +60,11 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t nCols; int32_t nTagCols; SSchemaWrapper *pSW; - STableMetaMsg * pTbMetaMsg; + STableMetaMsg * pTbMetaMsg = NULL; SSchema * pTagSchema; SRpcMsg rpcMsg; + int msgLen = 0; + int32_t code = TSDB_CODE_VND_APP_ERROR; pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); if (pTbCfg == NULL) { @@ -92,12 +94,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pTagSchema = NULL; } - int msgLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols); + msgLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols); pTbMetaMsg = (STableMetaMsg *)rpcMallocCont(msgLen); if (pTbMetaMsg == NULL) { goto _exit; } + memcpy(pTbMetaMsg->dbFname, pReq->dbFname, sizeof(pTbMetaMsg->dbFname)); strcpy(pTbMetaMsg->tbFname, pTbCfg->name); if (pTbCfg->type == META_CHILD_TABLE) { strcpy(pTbMetaMsg->stbFname, pStbCfg->name); @@ -120,13 +123,15 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { pSch->bytes = htonl(pSch->bytes); } + code = 0; + _exit: - rpcMsg.handle = pMsg->handle; - rpcMsg.ahandle = pMsg->ahandle; - rpcMsg.pCont = pTbMetaMsg; - rpcMsg.contLen = msgLen; - rpcMsg.code = 0; + rpcMsg.handle = pMsg->handle; + rpcMsg.ahandle = pMsg->ahandle; + rpcMsg.pCont = pTbMetaMsg; + rpcMsg.contLen = msgLen; + rpcMsg.code = code; rpcSendResponse(&rpcMsg); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 1e27749a1a..29f84ddcd0 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -248,9 +248,14 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { pOut->metaNum = 2; - - memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); - memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); + + if (pMetaMsg->dbFname[0]) { + snprintf(pOut->ctbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); + snprintf(pOut->tbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->stbFname); + } else { + memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); + memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); + } pOut->ctbMeta.vgId = pMetaMsg->vgId; pOut->ctbMeta.tableType = pMetaMsg->tableType; @@ -261,7 +266,11 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { } else { pOut->metaNum = 1; - memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); + if (pMetaMsg->dbFname[0]) { + snprintf(pOut->tbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); + } else { + memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); + } code = queryCreateTableMetaFromMsg(pMetaMsg, false, &pOut->tbMeta); } From d9cd1a7a0f74f22b1ed880e06910b349d316a527 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 07:32:02 +0000 Subject: [PATCH 63/97] fix table meta query problem --- include/dnode/vnode/meta/meta.h | 2 +- source/dnode/vnode/meta/src/metaBDBImpl.c | 101 ++++++++++------------ 2 files changed, 49 insertions(+), 54 deletions(-) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index d587107270..b8b8bad74a 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -38,7 +38,7 @@ typedef struct SMetaCfg { } SMetaCfg; typedef struct { - int32_t nCols; + uint32_t nCols; SSchema *pSchema; } SSchemaWrapper; diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index af8af6a052..4254ad0acd 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -23,6 +23,7 @@ typedef struct { tb_uid_t uid; int32_t sver; + int32_t padding; } SSchemaKey; struct SMetaDB { @@ -55,6 +56,8 @@ static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT * static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); +static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); +static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -169,18 +172,13 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { pBuf = buf; memset(&key, 0, sizeof(key)); memset(&value, 0, sizeof(key)); - SSchemaKey schemaKey = {uid, 0 /*TODO*/}; + SSchemaKey schemaKey = {uid, 0 /*TODO*/, 0}; key.data = &schemaKey; key.size = sizeof(schemaKey); - taosEncodeFixedU32(&pBuf, ncols); - for (size_t i = 0; i < ncols; i++) { - taosEncodeFixedI8(&pBuf, pSchema[i].type); - taosEncodeFixedI32(&pBuf, pSchema[i].colId); - taosEncodeFixedI32(&pBuf, pSchema[i].bytes); - taosEncodeString(&pBuf, pSchema[i].name); - } + SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema}; + metaEncodeSchema(&pBuf, &sw); value.data = buf; value.size = POINTER_DISTANCE(pBuf, buf); @@ -197,6 +195,38 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { } /* ------------------------ STATIC METHODS ------------------------ */ +static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { + int tlen = 0; + SSchema *pSchema; + + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeString(buf, pSchema->name); + } + + return tlen; +} + +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { + SSchema *pSchema; + + buf = taosDecodeFixedU32(buf, &pSW->nCols); + pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + + return buf; +} + static SMetaDB *metaNewDB() { SMetaDB *pDB = NULL; pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); @@ -376,15 +406,8 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { tsize += taosEncodeFixedU8(buf, pTbCfg->type); if (pTbCfg->type == META_SUPER_TABLE) { - tsize += taosEncodeVariantU32(buf, pTbCfg->stbCfg.nTagCols); - for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { - tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pTagSchema[i].type); - tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].colId); - tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].bytes); - tsize += taosEncodeString(buf, pTbCfg->stbCfg.pTagSchema[i].name); - } - - // tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); + SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema}; + tsize += metaEncodeSchema(buf, &sw); } else if (pTbCfg->type == META_CHILD_TABLE) { tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); @@ -403,14 +426,10 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeFixedU8(buf, &(pTbCfg->type)); if (pTbCfg->type == META_SUPER_TABLE) { - buf = taosDecodeVariantU32(buf, &(pTbCfg->stbCfg.nTagCols)); - pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols); - for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { - buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].bytes); - buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pTagSchema[i].name); - } + SSchemaWrapper sw; + buf = metaDecodeSchema(buf, &sw); + pTbCfg->stbCfg.nTagCols = sw.nCols; + pTbCfg->stbCfg.pTagSchema = sw.pSchema; } else if (pTbCfg->type == META_CHILD_TABLE) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); @@ -496,7 +515,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo int ret; void * pBuf; SSchema * pSchema; - SSchemaKey schemaKey = {uid, sver}; + SSchemaKey schemaKey = {uid, sver, 0}; DBT key = {0}; DBT value = {0}; @@ -507,38 +526,14 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo // Query ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0); if (ret != 0) { + printf("failed to query schema DB since %s================\n", db_strerror(ret)); return NULL; } // Decode the schema pBuf = value.data; - taosDecodeFixedI32(&pBuf, &nCols); - if (isinline) { - pSW = (SSchemaWrapper *)malloc(sizeof(*pSW) + sizeof(SSchema) * nCols); - if (pSW == NULL) { - return NULL; - } - pSW->pSchema = POINTER_SHIFT(pSW, sizeof(*pSW)); - } else { - pSW = (SSchemaWrapper *)malloc(sizeof(*pSW)); - if (pSW == NULL) { - return NULL; - } - - pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * nCols); - if (pSW->pSchema == NULL) { - free(pSW); - return NULL; - } - } - - for (int i = 0; i < nCols; i++) { - pSchema = pSW->pSchema + i; - taosDecodeFixedI8(&pBuf, &(pSchema->type)); - taosDecodeFixedI32(&pBuf, &(pSchema->colId)); - taosDecodeFixedI32(&pBuf, &(pSchema->bytes)); - taosDecodeStringTo(&pBuf, pSchema->name); - } + pSW = malloc(sizeof(*pSW)); + metaDecodeSchema(pBuf, pSW); return pSW; } From 54af08bc98f2111b953f34ce146a96878dd8c0c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 07:48:21 +0000 Subject: [PATCH 64/97] fix more --- include/common/tmsg.h | 6 +++--- include/dnode/vnode/meta/meta.h | 8 ++++---- source/dnode/vnode/meta/inc/metaTbCfg.h | 4 ---- source/libs/parser/src/dCDAstProcess.c | 3 +++ 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 39f50e0496..10aba94656 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1255,9 +1255,9 @@ typedef struct SVCreateTbReq { char* name; uint32_t ttl; uint32_t keep; -#define TD_SUPER_TABLE 0 -#define TD_CHILD_TABLE 1 -#define TD_NORMAL_TABLE 2 +#define TD_SUPER_TABLE TSDB_SUPER_TABLE +#define TD_CHILD_TABLE TSDB_CHILD_TABLE +#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE uint8_t type; union { struct { diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index b8b8bad74a..86ebb643a4 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -25,13 +25,13 @@ extern "C" { #endif +#define META_SUPER_TABLE TD_SUPER_TABLE +#define META_CHILD_TABLE TD_CHILD_TABLE +#define META_NORMAL_TABLE TD_NORMAL_TABLE + // Types exported typedef struct SMeta SMeta; -#define META_SUPER_TABLE 0 -#define META_CHILD_TABLE 1 -#define META_NORMAL_TABLE 2 - typedef struct SMetaCfg { /// LRU cache size uint64_t lruSize; diff --git a/source/dnode/vnode/meta/inc/metaTbCfg.h b/source/dnode/vnode/meta/inc/metaTbCfg.h index b4ee095967..b7b3924d14 100644 --- a/source/dnode/vnode/meta/inc/metaTbCfg.h +++ b/source/dnode/vnode/meta/inc/metaTbCfg.h @@ -22,10 +22,6 @@ extern "C" { #endif -#define META_SUPER_TABLE TD_SUPER_TABLE -#define META_CHILD_TABLE TD_CHILD_TABLE -#define META_NORMAL_TABLE TD_NORMAL_TABLE - int metaValidateTbCfg(SMeta *pMeta, const STbCfg *); size_t metaEncodeTbObjFromTbOptions(const STbCfg *, void *pBuf, size_t bsize); diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 024870014b..8c11f6e528 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -524,6 +524,9 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p } } + // TODO: serialize and + void *pBuf = NULL; + return TSDB_CODE_SUCCESS; } From 1c57b5e74675b1408d65674d685bb1f2f2fe583f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 08:30:40 +0000 Subject: [PATCH 65/97] batch create table --- include/common/tmsg.h | 4 +- source/common/src/tmsg.c | 29 ++++++++++++- source/dnode/vnode/impl/src/vnodeWrite.c | 17 ++++++-- source/libs/parser/src/dCDAstProcess.c | 55 ++++++++++++++++-------- 4 files changed, 82 insertions(+), 23 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 10aba94656..af250bff03 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1285,8 +1285,10 @@ typedef struct { int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); -int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq); +int tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); +int tSVCreateTbBatchReqSerialize(void** buf, SVCreateTbBatchReq* pReq); +void* tSVCreateTbBatchReqDeserialize(void* buf, SVCreateTbBatchReq* pReq); typedef struct SVCreateTbRsp { } SVCreateTbRsp; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b81143ee62..a18a472dba 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -98,7 +98,7 @@ int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) { return 0; } -int tSerializeSVCreateTbReq(void **buf, const SVCreateTbReq *pReq) { +int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int tlen = 0; tlen += taosEncodeFixedU64(buf, pReq->ver); @@ -193,6 +193,33 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { return buf; } +int tSVCreateTbBatchReqSerialize(void **buf, SVCreateTbBatchReq *pReq) { + int tlen = 0; + + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray)); + for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) { + SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i); + tlen += tSerializeSVCreateTbReq(buf, pCreateTbReq); + } + + return tlen; +} + +void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { + uint32_t nsize = 0; + + buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeFixedU32(buf, &nsize); + for (size_t i = 0; i < nsize; i++) { + SVCreateTbReq req; + buf = tDeserializeSVCreateTbReq(buf, &req); + taosArrayPush(pReq->pArray, &req); + } + + return buf; +} + /* ------------------------ STATIC METHODS ------------------------ */ static int tmsgStartEncode(SMsgEncoder *pME) { struct SMEListNode *pNode = (struct SMEListNode *)malloc(sizeof(*pNode)); diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 3b1442a02c..88a73ca174 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -27,7 +27,7 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { } int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg * pMsg; + SRpcMsg *pMsg; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); @@ -50,8 +50,9 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { } int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - SVCreateTbReq vCreateTbReq; - void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + SVCreateTbReq vCreateTbReq; + SVCreateTbBatchReq vCreateTbBatchReq; + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -68,7 +69,6 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { switch (pMsg->msgType) { case TDMT_VND_CREATE_STB: - case TDMT_VND_CREATE_TABLE: tDeserializeSVCreateTbReq(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbReq); if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { // TODO: handle error @@ -76,6 +76,15 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: maybe need to clear the requst struct break; + case TDMT_VND_CREATE_TABLE: + tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); + for (int i = 0; i < taosArrayGetSize(vCreateTbBatchReq.pArray); i++) { + SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); + if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { + // TODO: handle error + } + } + case TDMT_VND_DROP_STB: case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 8c11f6e528..ecea48f583 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -35,7 +35,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); SArray* array = NULL; - SName name = {0}; + SName name = {0}; tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); char dbFname[TSDB_DB_FNAME_LEN] = {0}; @@ -48,7 +48,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou pEpSet->numOfEps = info->numOfEps; pEpSet->inUse = info->inUse; - for(int32_t i = 0; i < pEpSet->numOfEps; ++i) { + for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); pEpSet->port[i] = info->epAddr[i].port; } @@ -190,7 +190,7 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { val = htonl(pCreate->numOfVgroups); if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); + TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); } return TSDB_CODE_SUCCESS; @@ -468,7 +468,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema}; - SToken* pItem = taosArrayGet(pValList, i); + SToken* pItem = taosArrayGet(pValList, i); code = parseValueToken(&endPtr, pItem, pSchema, tinfo.precision, tmpTokenBuf, KvRowAppend, ¶m, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { @@ -481,7 +481,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder); if (row == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_QRY_OUT_OF_MEMORY; } tdSortKVRowByColIdx(row); @@ -501,15 +501,15 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p req.ctbCfg.suid = pSuperTableMeta->suid; req.ctbCfg.pTag = row; -// pEpSet->inUse = info.inUse; -// pEpSet->numOfEps = info.numOfEps; -// for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { -// pEpSet->port[i] = info.epAddr[i].port; -// tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); -// } -// ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId); -// ((SMsgHead*)(*pOutput))->contLen = htonl(serLen); - SVgroupTablesBatch *pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId)); + // pEpSet->inUse = info.inUse; + // pEpSet->numOfEps = info.numOfEps; + // for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { + // pEpSet->port[i] = info.epAddr[i].port; + // tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + // } + // ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId); + // ((SMsgHead*)(*pOutput))->contLen = htonl(serLen); + SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId)); if (pTableBatch == NULL) { SVgroupTablesBatch tBatch = {0}; tBatch.info = info; @@ -518,14 +518,35 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p taosArrayPush(tBatch.req.pArray, &req); taosHashPut(pVgroupHashmap, &info.vgId, sizeof(info.vgId), &tBatch, sizeof(tBatch)); - } else { // add to the correct vgroup + } else { // add to the correct vgroup assert(info.vgId == pTableBatch->info.vgId); taosArrayPush(pTableBatch->req.pArray, &req); } } // TODO: serialize and - void *pBuf = NULL; + SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); + SVgroupTablesBatch** ppTbBatch = NULL; + do { + ppTbBatch = taosHashIterate(pVgroupHashmap, ppTbBatch); + if (ppTbBatch == NULL) break; + SVgroupTablesBatch* pTbBatch = *ppTbBatch; + + int tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req)); + void* buf = malloc(tlen); + if (buf == NULL) { + // TODO: handle error + } + + ((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId); + ((SMsgHead*)buf)->contLen = htonl(tlen); + + void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req)); + + taosArrayPush(pBufArray, &buf); + + } while (true); return TSDB_CODE_SUCCESS; } @@ -634,7 +655,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_SHOW: { SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, &pDcl->pExtension, pMsgBuf); - pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE)? TDMT_VND_SHOW_TABLES:TDMT_MND_SHOW; + pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE) ? TDMT_VND_SHOW_TABLES : TDMT_MND_SHOW; break; } From 14d83eda1ce7eaa073b855ea4a08acf648a57546 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 31 Dec 2021 16:41:50 +0800 Subject: [PATCH 66/97] feature/qnode --- source/libs/qcom/src/querymsg.c | 2 +- source/libs/scheduler/src/scheduler.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 29f84ddcd0..b50eb2c92d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -267,7 +267,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { pOut->metaNum = 1; if (pMetaMsg->dbFname[0]) { - snprintf(pOut->tbFname, "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); + snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); } else { memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3f1799507d..20eb94c2ff 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -664,7 +664,7 @@ int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) { } SSubQueryMsg *pMsg = msg; - + pMsg->sId = htobe64(schMgmt.sId); pMsg->queryId = htobe64(job->queryId); pMsg->taskId = htobe64(task->taskId); From a04f78c7602393d1d7b20c294a5d4375582f177a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 31 Dec 2021 09:41:04 +0000 Subject: [PATCH 67/97] mroe --- source/util/src/encode.c | 132 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 source/util/src/encode.c diff --git a/source/util/src/encode.c b/source/util/src/encode.c new file mode 100644 index 0000000000..9e648bb281 --- /dev/null +++ b/source/util/src/encode.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "encode.h" +#include "freelist.h" + +#define CODER_NODE_FIELDS \ + uint8_t* data; \ + int32_t size; \ + int32_t pos; + +struct SCoderNode { + TD_SLIST_NODE(SCoderNode); + CODER_NODE_FIELDS +}; + +typedef struct { + td_endian_t endian; + SFreeList fl; + CODER_NODE_FIELDS + TD_SLIST(SCoderNode) stack; +} SCoder; + +bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } + +void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size) { + pCoder->endian = endian; + pCoder->data = data; + pCoder->size = size; + pCoder->pos = 0; + tFreeListInit(&(pCoder->fl)); + TD_SLIST_INIT(&(pCoder->stack)); +} + +void tCoderClear(SCoder* pCoder) { + tFreeListClear(&(pCoder->fl)); + struct SCoderNode* pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pCoder->stack)); + if (pNode == NULL) break; + TD_SLIST_POP(&(pCoder->stack)); + free(pNode); + } +} + +int tStartEncode(SCoder* pCoder) { + if (pCoder->data) { + struct SCoderNode* pNode = malloc(sizeof(*pNode)); + if (pNode == NULL) return -1; + + pNode->data = pCoder->data; + pNode->pos = pCoder->pos; + pNode->size = pCoder->size; + + pCoder->data = pNode->data + sizeof(int32_t); + pCoder->pos = 0; + pCoder->size = pNode->size - pNode->pos - sizeof(int32_t); + + TD_SLIST_PUSH(&(pCoder->stack), pNode); + } else { + pCoder->pos += sizeof(int32_t); + } + return 0; +} + +void tEndEncode(SCoder* pCoder) { + struct SCoderNode* pNode; + + if (pCoder->data) { + pNode = TD_SLIST_HEAD(&(pCoder->stack)); + ASSERT(pNode); + TD_SLIST_POP(&(pCoder->stack)); + + // TODO: tEncodeI32(pNode, pCoder->pos); + + pCoder->data = pNode->data; + pCoder->size = pNode->size; + pCoder->pos = pNode->pos + pCoder->pos; + + free(pNode); + } +} + +int tStartDecode(SCoder* pCoder) { + int32_t size; + struct SCoderNode* pNode; + + pNode = malloc(sizeof(*pNode)); + if (pNode == NULL) return -1; + + // TODO: tDecodeI32(pCoder, &size); + + pNode->data = pCoder->data; + pNode->pos = pCoder->pos; + pNode->size = pCoder->size; + + pCoder->data = pCoder->data; + pCoder->size = size; + pCoder->pos = 0; + + TD_SLIST_PUSH(&(pCoder->stack), pNode); + + return 0; +} + +void tEndDecode(SCoder* pCoder) { + ASSERT(tDecodeIsEnd(pCoder)); + + struct SCoderNode* pNode; + + pNode = TD_SLIST_HEAD(&(pCoder->stack)); + ASSERT(pNode); + TD_SLIST_POP(&(pCoder->stack)); + + pCoder->data = pNode->data; + pCoder->size = pNode->size; + pCoder->pos = pCoder->pos + pNode->pos; + + free(pNode); +} From 9920b43be5997caadbd15c9745da7b265a5b59ed Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 31 Dec 2021 18:06:13 +0800 Subject: [PATCH 68/97] fix fst bug --- source/libs/index/src/index.c | 2 + source/libs/index/src/index_cache.c | 5 +- source/libs/index/src/index_fst.c | 4 +- source/libs/index/src/index_tfile.c | 6 +- source/libs/index/test/fstTest.cc | 115 ++++++++++++++++++++++++++- source/libs/index/test/indexTests.cc | 15 +--- 6 files changed, 127 insertions(+), 20 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 5167196031..6398259a96 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -360,6 +360,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { if (sz > 0) { // TODO(yihao): remove duplicate tableid TFileValue* lv = taosArrayGetP(result, sz - 1); + // indexError("merge colVal: %s", lv->colVal); if (strcmp(lv->colVal, tv->colVal) == 0) { taosArrayAddAll(lv->tableId, tv->tableId); tfileValueDestroy(tv); @@ -368,6 +369,7 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv) { } } else { taosArrayPush(result, &tv); + // indexError("merge colVal: %s", tv->colVal); } } static void indexDestroyTempResult(SArray* result) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 0e46445a00..503d7cd928 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 10000 * 10 +#define MEM_TERM_LIMIT 10 * 10000 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -353,6 +353,9 @@ static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; if (iter == NULL) { return false; } IterateValue* iv = &itera->val; + if (iv->colVal != NULL && iv->val != NULL) { + // indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + } iterateValueDestroy(iv, false); bool next = tSkipListIterNext(iter); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 04a08dafd2..088c7369d5 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -319,7 +319,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; - COMMON_INDEX(inp, 0x111111, val); + COMMON_INDEX(inp, 0b111111, val); s->val = (s->val & fstStateDict[s->state].val) | val; } @@ -369,7 +369,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) { bool null = false; uint8_t inp = fstStateCommInput(s, &null); uint8_t* data = fstSliceData(slice, NULL); - return null == false ? inp : data[-1]; + return null == false ? inp : data[node->start - 1]; } uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index fc4f8593a1..6669198861 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -385,8 +385,10 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { } static bool tfileIteratorNext(Iterate* iiter) { IterateValue* iv = &iiter->val; + if (iv->colVal != NULL && iv->val != NULL) { + // indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val)); + } iterateValueDestroy(iv, false); - // SArray* tblIds = iv->val; char* colVal = NULL; uint64_t offset = 0; @@ -406,7 +408,7 @@ static bool tfileIteratorNext(Iterate* iiter) { if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; - + return true; // std::string key(ch, sz); } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index da974ce6c4..3d978c05a5 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -24,8 +24,13 @@ class FstWriter { _b = fstBuilderCreate(_wc, 0); } bool Put(const std::string& key, uint64_t val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); bool ok = fstBuilderInsert(_b, skey, val); + fstSliceDestroy(&skey); return ok; } @@ -61,6 +66,11 @@ class FstReadMemory { return _fst != NULL; } bool Get(const std::string& key, uint64_t* val) { + // char buf[128] = {0}; + // int len = 0; + // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len); + // FstSlice skey = fstSliceCreate((uint8_t*)buf, len); + FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size()); bool ok = fstGet(_fst, &skey, val); fstSliceDestroy(&skey); @@ -135,15 +145,109 @@ int Performance_fstWriteRecords(FstWriter* b) { } return L * M * N; } +void Performance_fstReadRecords(FstReadMemory* m) { + std::string str("aa"); + for (int i = 0; i < M; i++) { + str[0] = 'a' + i; + str.resize(2); + for (int j = 0; j < N; j++) { + str[1] = 'a' + j; + str.resize(2); + for (int k = 0; k < L; k++) { + str.push_back('a'); + uint64_t val, cost; + if (m->GetWithTimeCostUs(str, &val, &cost)) { + printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost); + } else { + printf("failed to get key: %s\n", str.c_str()); + } + } + } + } +} + +void checkMillonWriteAndReadOfFst() { + tfInit(); + FstWriter* fw = new FstWriter; + Performance_fstWriteRecords(fw); + delete fw; + FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); + + if (fr->init()) { printf("success to init fst read"); } + + Performance_fstReadRecords(fr); + tfCleanup(); + delete fr; +} +void checkFstLongTerm() { + tfInit(); + FstWriter* fw = new FstWriter; + // Performance_fstWriteRecords(fw); + + fw->Put("A B", 1); + fw->Put("C", 2); + fw->Put("a", 3); + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + { + uint64_t val = 0; + if (m->Get("A B", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + { + uint64_t val = 0; + if (m->Get("C", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + { + uint64_t val = 0; + if (m->Get("a", &val)) { + std::cout << "success to Get: " << val << std::endl; + } else { + std::cout << "failed to Get:" << val << std::endl; + } + } + + // prefix search + // std::vector result; + + // AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + // m->Search(ctx, result); + // std::cout << "size: " << result.size() << std::endl; + // assert(result.size() == count); + // for (int i = 0; i < result.size(); i++) { + // assert(result[i] == i); // check result + //} + tfCleanup(); + // free(ctx); + // delete m; +} void checkFstCheckIterator() { tfInit(); FstWriter* fw = new FstWriter; int64_t s = taosGetTimestampUs(); int count = 2; - Performance_fstWriteRecords(fw); + // Performance_fstWriteRecords(fw); int64_t e = taosGetTimestampUs(); std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + + fw->Put("Hello world", 1); + fw->Put("hello world", 2); + fw->Put("hello worle", 3); + fw->Put("hello worlf", 4); delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); @@ -171,7 +275,7 @@ void checkFstCheckIterator() { void fst_get(Fst* fst) { for (int i = 0; i < 10000; i++) { - std::string term = "Hello"; + std::string term = "Hello World"; FstSlice key = fstSliceCreate((uint8_t*)term.c_str(), term.size()); uint64_t offset = 0; bool ret = fstGet(fst, &key, &offset); @@ -189,7 +293,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; - TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1"); + TFileReader* reader = tfileReaderOpen(arg, 0, 999992, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { threads[i] = std::thread(fst_get, reader->fst); @@ -203,9 +307,12 @@ void validateTFile(char* arg) { tfCleanup(); } int main(int argc, char* argv[]) { - if (argc > 1) { validateTFile(argv[1]); } + // tool to check all kind of fst test + // if (argc > 1) { validateTFile(argv[1]); } // checkFstCheckIterator(); + // checkFstLongTerm(); // checkFstPrefixSearch(); + checkMillonWriteAndReadOfFst(); return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 080becccf1..3ad64cd03e 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -787,14 +787,15 @@ TEST_F(IndexEnv2, testIndexOpen) { } TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp/test"; + std::string path = "/tmp/test1"; if (index->Init(path) != 0) { // r std::cout << "failed to init" << std::endl; } int numOfTable = 100 * 10000; - index->WriteMillonData("tag1", "Hello", numOfTable); - int target = index->SearchOne("tag1", "Hello"); + index->WriteMillonData("tag1", "Hello Wolrd", numOfTable); + int target = index->SearchOne("tag1", "Hello Wolrd"); + std::cout << "Get Index: " << target << std::endl; assert(numOfTable == target); } @@ -821,14 +822,6 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { threads[i].join(); } } -TEST_F(IndexEnv2, testIndex_multi_thread_write) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} -} -TEST_F(IndexEnv2, testIndex_multi_thread_read) { - std::string path = "/tmp"; - if (index->Init(path) != 0) {} -} TEST_F(IndexEnv2, testIndex_restart) { std::string path = "/tmp"; From f0044d826f1d5814d106bfa4cb3529f36f8b5b76 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Dec 2021 18:57:57 +0800 Subject: [PATCH 69/97] [td-11818]support create multiple tables. --- include/common/tmsgtype.h | 3 +- include/libs/parser/parsenodes.h | 4 +- source/client/src/clientImpl.c | 2 +- source/libs/parser/inc/parserInt.h | 4 +- source/libs/parser/src/dCDAstProcess.c | 139 ++++++++++++++++-------- source/libs/parser/src/parser.c | 29 +++-- source/libs/parser/test/parserTests.cpp | 10 +- source/libs/planner/inc/plannerInt.h | 2 +- source/libs/planner/src/logicPlan.c | 15 ++- source/libs/planner/src/physicalPlan.c | 22 ++-- 10 files changed, 151 insertions(+), 79 deletions(-) diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 8e7ad87a0a..ebbf99b942 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -40,8 +40,9 @@ enum { // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_STABLE, "create-stable" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_FUNCTION, "drop-function" ) diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 041adbb582..b326ac032c 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -27,7 +27,7 @@ extern "C" { #include "tname.h" #include "tvariant.h" -/* +/** * The first field of a node of any type is guaranteed to be the int16_t. * Hence the type of any node can be gotten by casting it to SQueryNode. */ @@ -157,7 +157,7 @@ typedef struct SVgDataBlocks { typedef struct SInsertStmtInfo { int16_t nodeType; SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not + int8_t schemaAttache; // denote if submit block is built with table schema or not uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] const char* sql; // current sql statement position diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 050e763919..b0b2c57ee4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -202,7 +202,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQuery, SQueryDag** pDag) { } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { - if (TSDB_SQL_INSERT == pRequest->type) { + if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { return scheduleExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob, &pRequest->affectedRows); } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 4bbe6ab907..346bd0cbe4 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -68,7 +68,9 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ * @param type * @return */ -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, int32_t msgBufLen); +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); + +SInsertStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index ecea48f583..76e26c159b 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -326,8 +326,7 @@ typedef struct SVgroupTablesBatch { SVgroupInfo info; } SVgroupTablesBatch; -int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len, - SEpSet* pEpSet) { +int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { const char* msg1 = "invalid table name"; const char* msg2 = "tags number not matched"; const char* msg3 = "tag value too long"; @@ -359,7 +358,11 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p size_t numOfInputTag = taosArrayGetSize(pValList); STableMeta* pSuperTableMeta = NULL; - catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta); + code = catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &name, &pSuperTableMeta); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + assert(pSuperTableMeta != NULL); // too long tag values will return invalid sql, not be truncated automatically @@ -501,14 +504,6 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p req.ctbCfg.suid = pSuperTableMeta->suid; req.ctbCfg.pTag = row; - // pEpSet->inUse = info.inUse; - // pEpSet->numOfEps = info.numOfEps; - // for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - // pEpSet->port[i] = info.epAddr[i].port; - // tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - // } - // ((SMsgHead*)(*pOutput))->vgId = htonl(info.vgId); - // ((SMsgHead*)(*pOutput))->contLen = htonl(serLen); SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId)); if (pTableBatch == NULL) { SVgroupTablesBatch tBatch = {0}; @@ -525,12 +520,12 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p } // TODO: serialize and - SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); - SVgroupTablesBatch** ppTbBatch = NULL; + SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); + + SVgroupTablesBatch* pTbBatch = NULL; do { - ppTbBatch = taosHashIterate(pVgroupHashmap, ppTbBatch); - if (ppTbBatch == NULL) break; - SVgroupTablesBatch* pTbBatch = *ppTbBatch; + pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch); + if (pTbBatch == NULL) break; int tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req)); void* buf = malloc(tlen); @@ -544,17 +539,29 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req)); - taosArrayPush(pBufArray, &buf); + SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks)); + pVgData->vg = pTbBatch->info; + pVgData->pData = buf; + pVgData->size = tlen; + pVgData->numOfTables = (int32_t) taosArrayGetSize(pTbBatch->req.pArray); + taosArrayPush(pBufArray, &pVgData); } while (true); + SInsertStmtInfo* pStmtInfo = calloc(1, sizeof(SInsertStmtInfo)); + pStmtInfo->nodeType = TSDB_SQL_CREATE_TABLE; + pStmtInfo->pDataBlocks = pBufArray; + *pOutput = pStmtInfo; + *len = sizeof(SInsertStmtInfo); + return TSDB_CODE_SUCCESS; } -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, - int32_t msgBufLen) { +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; + SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf* pMsgBuf = &m; @@ -571,21 +578,25 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + code = buildInvalidOperationMsg(pMsgBuf, msg3); + goto _error; } if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + code = buildInvalidOperationMsg(pMsgBuf, msg2); + goto _error; } if (pInfo->type == TSDB_SQL_CREATE_USER) { if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } } else { if (pUser->type == TSDB_ALTER_USER_PASSWD) { if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); @@ -596,10 +607,12 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) { // pCmd->count = 2; } else { - return buildInvalidOperationMsg(pMsgBuf, msg4); + code = buildInvalidOperationMsg(pMsgBuf, msg4); + goto _error; } } else { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } } @@ -618,15 +631,18 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SToken* pPwd = &pInfo->pMiscInfo->user.passwd; if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } if (pName->n >= TSDB_USER_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + code = buildInvalidOperationMsg(pMsgBuf, msg3); + goto _error; } if (parserValidateNameToken(pName) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + code = buildInvalidOperationMsg(pMsgBuf, msg2); + goto _error; } SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; @@ -636,7 +652,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { } else { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } } @@ -655,6 +672,10 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_SHOW: { SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, &pDcl->pExtension, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE) ? TDMT_VND_SHOW_TABLES : TDMT_MND_SHOW; break; } @@ -664,13 +685,15 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (parserValidateNameToken(pToken) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg); + code = buildInvalidOperationMsg(pMsgBuf, msg); + goto _error; } SName n = {0}; int32_t ret = tNameSetDbName(&n, pCtx->acctId, pToken->z, pToken->n); if (ret != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg); + code = buildInvalidOperationMsg(pMsgBuf, msg); + goto _error; } SUseDbMsg* pUseDbMsg = (SUseDbMsg*)calloc(1, sizeof(SUseDbMsg)); @@ -689,19 +712,22 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + code = buildInvalidOperationMsg(pMsgBuf, msg2); + goto _error; } char buf[TSDB_DB_NAME_LEN] = {0}; SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + code = TSDB_CODE_TSC_INVALID_OPERATION; + goto _error; } pDcl->pMsg = (char*)pCreateMsg; @@ -719,7 +745,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm SName name = {0}; code = tNameSetDbName(&name, pCtx->acctId, dbName->z, dbName->n); if (code != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + code = buildInvalidOperationMsg(pMsgBuf, msg1); + goto _error; } SDropDbMsg* pDropDbMsg = (SDropDbMsg*)calloc(1, sizeof(SDropDbMsg)); @@ -731,7 +758,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm pDcl->msgType = TDMT_MND_DROP_DB; pDcl->msgLen = sizeof(SDropDbMsg); pDcl->pMsg = (char*)pDropDbMsg; - return TSDB_CODE_SUCCESS; + break; } case TSDB_SQL_CREATE_TABLE: { @@ -739,14 +766,16 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { - return code; + terrno = code; + goto _error; } + pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf); pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB; } else if (pCreateTable->type == TSQL_CREATE_CTABLE) { - if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet)) != + if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen)) != TSDB_CODE_SUCCESS) { - return code; + goto _error; } pDcl->msgType = TDMT_VND_CREATE_TABLE; @@ -761,7 +790,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_DROP_TABLE: { pDcl->pMsg = (char*)buildDropStableMsg(pInfo, &pDcl->msgLen, pCtx, pMsgBuf); if (pDcl->pMsg == NULL) { - code = terrno; + goto _error; } pDcl->msgType = TDMT_MND_DROP_STB; @@ -771,7 +800,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_CREATE_DNODE: { pDcl->pMsg = (char*)buildCreateDnodeMsg(pInfo, &pDcl->msgLen, pMsgBuf); if (pDcl->pMsg == NULL) { - code = terrno; + goto _error; } pDcl->msgType = TDMT_MND_CREATE_DNODE; @@ -781,7 +810,7 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm case TSDB_SQL_DROP_DNODE: { pDcl->pMsg = (char*)buildDropDnodeMsg(pInfo, &pDcl->msgLen, pMsgBuf); if (pDcl->pMsg == NULL) { - code = terrno; + goto _error; } pDcl->msgType = TDMT_MND_DROP_DNODE; @@ -792,5 +821,29 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm break; } - return code; + return pDcl; + + _error: + terrno = code; + tfree(pDcl); + return NULL; } + +SInsertStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + assert(pCreateTable->type == TSQL_CREATE_CTABLE); + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf* pMsgBuf = &m; + + SInsertStmtInfo* pInsertStmt = NULL; + + int32_t msgLen = 0; + int32_t code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, (char**) &pInsertStmt, &msgLen); + if (code != TSDB_CODE_SUCCESS) { + tfree(pInsertStmt); + return NULL; + } + + return pInsertStmt; +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 710cf4b5d0..1b4d05808c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -32,7 +32,7 @@ bool isInsertSql(const char* pStr, size_t length) { } bool qIsDdlQuery(const SQueryNode* pQuery) { - return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type; + return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type && TSDB_SQL_CREATE_TABLE != pQuery->type; } int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { @@ -44,16 +44,29 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { } if (!isDqlSqlStatement(&info)) { - SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); - if (NULL == pDcl) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. - return terrno; + bool toVnode = false; + if (info.type == TSDB_SQL_CREATE_TABLE) { + SCreateTableSql* pCreateSql = info.pCreateTableInfo; + if (pCreateSql->type == TSQL_CREATE_CTABLE || pCreateSql->type == TSQL_CREATE_TABLE) { + toVnode = true; + } } - pDcl->nodeType = info.type; - int32_t code = qParserValidateDclSqlNode(&info, &pCxt->ctx, pDcl, pCxt->pMsg, pCxt->msgLen); - if (code == TSDB_CODE_SUCCESS) { + if (toVnode) { + SInsertStmtInfo *pInsertInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + if (pInsertInfo == NULL) { + return terrno; + } + + *pQuery = (SQueryNode*) pInsertInfo; + } else { + SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + if (pDcl == NULL) { + return terrno; + } + *pQuery = (SQueryNode*)pDcl; + pDcl->nodeType = info.type; } } else { SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index a67a9a8be8..fe430c5f5e 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -714,10 +714,9 @@ TEST(testCase, show_user_Test) { SSqlInfo info1 = doGenerateAST(sql1); ASSERT_EQ(info1.valid, true); - SDclStmtInfo output; SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; - int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len); - ASSERT_EQ(code, 0); + SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); + ASSERT_NE(output, nullptr); // convert the show command to be the select query // select name, privilege, create_time, account from information_schema.users; @@ -735,10 +734,9 @@ TEST(testCase, create_user_Test) { ASSERT_EQ(info1.valid, true); ASSERT_EQ(isDclSqlStatement(&info1), true); - SDclStmtInfo output; SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"}; - int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len); - ASSERT_EQ(code, 0); + SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); + ASSERT_NE(output, nullptr); destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 35c6d59ffe..a68102ea6e 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -40,7 +40,7 @@ extern "C" { #define QNODE_SESSIONWINDOW 12 #define QNODE_STATEWINDOW 13 #define QNODE_FILL 14 -#define QNODE_INSERT 15 +#define QNODE_MODIFY 15 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 136073aa60..5f95f86d4a 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -37,15 +37,19 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } -int32_t createInsertPlan(const SInsertStmtInfo* pInsert, SQueryPlanNode** pQueryPlan) { +static int32_t createInsertPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { + SInsertStmtInfo* pInsert = (SInsertStmtInfo*)pNode; + *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); if (NULL == *pQueryPlan || NULL == blocks) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - (*pQueryPlan)->info.type = QNODE_INSERT; + + (*pQueryPlan)->info.type = QNODE_MODIFY; taosArrayAddAll(blocks, pInsert->pDataBlocks); (*pQueryPlan)->pExtInfo = blocks; + return TSDB_CODE_SUCCESS; } @@ -62,13 +66,14 @@ int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { case TSDB_SQL_SELECT: { return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); } + case TSDB_SQL_INSERT: - return createInsertPlan((const SInsertStmtInfo*)pNode, pQueryPlan); + case TSDB_SQL_CREATE_TABLE: + return createInsertPlan(pNode, pQueryPlan); + default: return TSDB_CODE_FAILED; } - - return TSDB_CODE_SUCCESS; } int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 97c9cec7c7..978b1554f3 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -34,7 +34,7 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; -static void* vailidPointer(void* p) { +static void* validPointer(void* p) { if (NULL == p) { THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); } @@ -76,7 +76,7 @@ int32_t dsinkNameToDsinkType(const char* name) { } static SDataSink* initDataSink(int32_t type, int32_t size) { - SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size)); + SDataSink* sink = (SDataSink*)validPointer(calloc(1, size)); sink->info.type = type; sink->info.name = dsinkTypeToDsinkName(type); return sink; @@ -121,7 +121,7 @@ static bool cloneExprArray(SArray** dst, SArray* src) { } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { - SPhyNode* node = (SPhyNode*)vailidPointer(calloc(1, size)); + SPhyNode* node = (SPhyNode*)validPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { @@ -184,7 +184,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { - SSubplan* subplan = vailidPointer(calloc(1, sizeof(SSubplan))); + SSubplan* subplan = validPointer(calloc(1, sizeof(SSubplan))); subplan->id = pCxt->nextId; ++(pCxt->nextId.subplanId); subplan->type = type; @@ -192,15 +192,15 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL != pCxt->pCurrentSubplan) { subplan->level = pCxt->pCurrentSubplan->level + 1; if (NULL == pCxt->pCurrentSubplan->pChildern) { - pCxt->pCurrentSubplan->pChildern = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + pCxt->pCurrentSubplan->pChildern = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); } taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); - subplan->pParents = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + subplan->pParents = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } SArray* currentLevel; if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { - currentLevel = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + currentLevel = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); } else { currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); @@ -272,7 +272,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; - case QNODE_INSERT: + case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; default: @@ -306,7 +306,7 @@ static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { } static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { - if (QNODE_INSERT == pRoot->info.type) { + if (QNODE_MODIFY == pRoot->info.type) { splitInsertSubplan(pCxt, pRoot); } else { SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); @@ -321,12 +321,12 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, - .pDag = vailidPointer(calloc(1, sizeof(SQueryDag))), + .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, .nextId = {0} // todo queryid }; *pDag = context.pDag; - context.pDag->pSubplans = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + context.pDag->pSubplans = validPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); createSubplanByLevel(&context, pQueryNode); } CATCH(code) { CLEANUP_EXECUTE(); From db474b20789740c325b4094f6af96a583672338e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 2 Jan 2022 14:28:38 +0800 Subject: [PATCH 70/97] refactor code --- source/libs/index/inc/indexInt.h | 9 ++++ source/libs/index/inc/index_cache.h | 3 +- source/libs/index/inc/index_tfile.h | 13 ++--- source/libs/index/inc/index_util.h | 2 +- source/libs/index/src/index.c | 59 ++++++++++++++++------ source/libs/index/src/index_cache.c | 4 +- source/libs/index/src/index_tfile.c | 73 +++++++++++++--------------- source/libs/index/test/indexTests.cc | 2 +- 8 files changed, 94 insertions(+), 71 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 048c9e804e..6e1256d857 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -108,8 +108,17 @@ void iterateValueDestroy(IterateValue* iv, bool destroy); extern void* indexQhandle; +typedef struct TFileCacheKey { + uint64_t suid; + uint8_t colType; + char* colName; + int32_t nColName; +} ICacheKey; + int indexFlushCacheTFile(SIndex* sIdx, void*); +int32_t indexSerialCacheKey(ICacheKey* key, char* buf); + #define indexFatal(...) \ do { \ if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); } \ diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 12b66bca2c..805137ccaf 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -42,6 +42,7 @@ typedef struct IndexCache { int32_t version; int32_t nTerm; int8_t type; + uint64_t suid; pthread_mutex_t mtx; } IndexCache; @@ -58,7 +59,7 @@ typedef struct CacheTerm { } CacheTerm; // -IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type); +IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type); void indexCacheDestroy(void* cache); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 4928e01a63..675203fa1a 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -49,13 +49,6 @@ typedef struct TFileValue { int32_t offset; } TFileValue; -typedef struct TFileCacheKey { - uint64_t suid; - uint8_t colType; - char* colName; - int32_t nColName; -} TFileCacheKey; - // table cache // refactor to LRU cache later typedef struct TFileCache { @@ -103,10 +96,10 @@ typedef struct TFileReaderOpt { // tfile cache, manage tindex reader TFileCache* tfileCacheCreate(const char* path); void tfileCacheDestroy(TFileCache* tcache); -TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); -void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); +TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key); +void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader); -TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName); +TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName); TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName); TFileReader* tfileReaderCreate(WriterCtx* ctx); diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index 21c5ca155b..adeb52bb8c 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -34,7 +34,7 @@ extern "C" { #define SERIALIZE_VAR_TO_BUF(buf, var, type) \ do { \ type c = var; \ - assert(sizeof(var) == sizeof(type)); \ + assert(sizeof(type) == sizeof(c)); \ memcpy((void*)buf, (void*)&c, sizeof(c)); \ buf += sizeof(c); \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 6398259a96..11a2aa5e5f 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -17,6 +17,7 @@ #include "indexInt.h" #include "index_cache.h" #include "index_tfile.h" +#include "index_util.h" #include "tdef.h" #include "tsched.h" @@ -130,18 +131,28 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { // TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm* p = taosArrayGetP(fVals, i); - IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName}; + int32_t sz = indexSerialCacheKey(&key, buf); + + IndexCache** cache = taosHashGet(index->colObj, buf, sz); if (cache == NULL) { - IndexCache* pCache = indexCacheCreate(index, p->colName, p->colType); - taosHashPut(index->colObj, p->colName, p->nColName, &pCache, sizeof(void*)); + IndexCache* pCache = indexCacheCreate(index, p->suid, p->colName, p->colType); + taosHashPut(index->colObj, buf, sz, &pCache, sizeof(void*)); } } pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm* p = taosArrayGetP(fVals, i); - IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + + char buf[128] = {0}; + ICacheKey key = {.suid = p->suid, .colName = p->colName}; + int32_t sz = indexSerialCacheKey(&key, buf); + + IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); if (ret != 0) { return ret; } @@ -296,7 +307,12 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result // Get col info IndexCache* cache = NULL; pthread_mutex_lock(&sIdx->mtx); - IndexCache** pCache = taosHashGet(sIdx->colObj, colName, nColName); + + char buf[128] = {0}; + ICacheKey key = {.suid = term->suid, .colName = term->colName}; + int32_t sz = indexSerialCacheKey(&key, buf); + + IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); if (pCache == NULL) { pthread_mutex_unlock(&sIdx->mtx); return -1; @@ -385,10 +401,12 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; - TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); + TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); + if (pReader == NULL) { indexWarn("empty pReader found"); } // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); + if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } SArray* result = taosArrayInit(1024, sizeof(void*)); @@ -468,7 +486,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { int32_t version = CACHE_VERSION(cache); uint8_t colType = cache->type; - TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, cache->colName, colType); + TFileWriter* tw = tfileWriterOpen(sIdx->path, cache->suid, version, cache->colName, colType); if (tw == NULL) { indexError("failed to open file to write"); return -1; @@ -481,14 +499,13 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { } tfileWriterClose(tw); - TFileReader* reader = tfileReaderOpen(sIdx->path, sIdx->suid, version, cache->colName); + TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); + + char buf[128] = {0}; + TFileHeader* header = &reader->header; + ICacheKey key = { + .suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; - char buf[128] = {0}; - TFileHeader* header = &reader->header; - TFileCacheKey key = {.suid = header->suid, - .colName = header->colName, - .nColName = strlen(header->colName), - .colType = header->colType}; pthread_mutex_lock(&sIdx->mtx); IndexTFile* ifile = (IndexTFile*)sIdx->tindex; @@ -499,3 +516,13 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { END: tfileWriterClose(tw); } + +int32_t indexSerialCacheKey(ICacheKey* key, char* buf) { + char* p = buf; + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + // SERIALIZE_MEM_TO_BUF(buf, key, colType); + // SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); + return buf - p; +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 503d7cd928..0f00d9d4af 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -40,7 +40,7 @@ static bool indexCacheIteratorNext(Iterate* itera); static IterateValue* indexCacheIteratorGetValue(Iterate* iter); -IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { +IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) { IndexCache* cache = calloc(1, sizeof(IndexCache)); if (cache == NULL) { indexError("failed to create index cache"); @@ -53,7 +53,7 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) { cache->type = type; cache->index = idx; cache->version = 0; - + cache->suid = suid; pthread_mutex_init(&cache->mtx, NULL); indexCacheRef(cache); return cache; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 6669198861..fe54b5ebb3 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -51,7 +51,6 @@ static void tfileDestroyFileName(void* elem); static int tfileCompare(const void* a, const void* b); static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); -static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); @@ -80,18 +79,18 @@ TFileCache* tfileCacheCreate(const char* path) { goto End; } - char buf[128] = {0}; - TFileReader* reader = tfileReaderCreate(wc); - TFileHeader* header = &reader->header; - TFileCacheKey key = {.suid = header->suid, - .colName = header->colName, - .nColName = strlen(header->colName), - .colType = header->colType}; - tfileSerialCacheKey(&key, buf); + char buf[128] = {0}; + TFileReader* reader = tfileReaderCreate(wc); + TFileHeader* header = &reader->header; + ICacheKey key = {.suid = header->suid, + .colName = header->colName, + .nColName = strlen(header->colName), + .colType = header->colType}; + int32_t sz = indexSerialCacheKey(&key, buf); + assert(sz < sizeof(buf)); + taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); - // indexTable - taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } taosArrayDestroyEx(files, tfileDestroyFileName); return tcache; @@ -117,30 +116,30 @@ void tfileCacheDestroy(TFileCache* tcache) { free(tcache); } -TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { - char buf[128] = {0}; - tfileSerialCacheKey(key, buf); - - TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); +TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) { + char buf[128] = {0}; + int32_t sz = indexSerialCacheKey(key, buf); + assert(sz < sizeof(buf)); + TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); if (reader == NULL) { return NULL; } tfileReaderRef(*reader); return *reader; } -void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { - char buf[128] = {0}; - tfileSerialCacheKey(key, buf); +void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) { + char buf[128] = {0}; + int32_t sz = indexSerialCacheKey(key, buf); // remove last version index reader - TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); + TFileReader** p = taosHashGet(tcache->tableCache, buf, sz); if (p != NULL) { TFileReader* oldReader = *p; - taosHashRemove(tcache->tableCache, buf, strlen(buf)); + taosHashRemove(tcache->tableCache, buf, sz); oldReader->remove = true; tfileReaderUnRef(oldReader); } + taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); tfileReaderRef(reader); - taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); return; } TFileReader* tfileReaderCreate(WriterCtx* ctx) { @@ -230,8 +229,6 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c TFileReader* reader = tfileReaderCreate(wc); return reader; - - // tfileSerialCacheKey(&key, buf); } TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { // char pathBuf[128] = {0}; @@ -325,15 +322,19 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { tfileWriterClose(tw); return -1; } - // write fst + + // write data indexError("--------Begin----------------"); for (size_t i = 0; i < sz; i++) { // TODO, fst batch write later TFileValue* v = taosArrayGetP((SArray*)data, i); - if (tfileWriteData(tw, v) == 0) { - // + if (tfileWriteData(tw, v) != 0) { + indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset, + (int)taosArrayGetSize(v->tableId)); + } else { + indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, + (int)taosArrayGetSize(v->tableId)); } - indexError("data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); } indexError("--------End----------------"); fstBuilderFinish(tw->fb); @@ -369,9 +370,8 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; - SIndexTerm* term = query->term; - TFileCacheKey key = { - .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; + SIndexTerm* term = query->term; + ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); if (reader == NULL) { return 0; } @@ -453,9 +453,9 @@ void tfileIteratorDestroy(Iterate* iter) { free(iter); } -TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { +TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { if (tf == NULL) { return NULL; } - TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; + ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } @@ -650,10 +650,3 @@ static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, } return -1; } -static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { - // SERIALIZE_MEM_TO_BUF(buf, key, suid); - // SERIALIZE_VAR_TO_BUF(buf, '_', char); - // SERIALIZE_MEM_TO_BUF(buf, key, colType); - // SERIALIZE_VAR_TO_BUF(buf, '_', char); - SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); -} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 3ad64cd03e..7bf40bc7b3 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -477,7 +477,7 @@ class CacheObj { public: CacheObj() { // TODO - cache = indexCacheCreate(NULL, "voltage", TSDB_DATA_TYPE_BINARY); + cache = indexCacheCreate(NULL, 0, "voltage", TSDB_DATA_TYPE_BINARY); } int Put(SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { int ret = indexCachePut(cache, term, uid); From 734ff3f83938a2adb69e34c78aac6b61e080e949 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 06:48:48 +0000 Subject: [PATCH 71/97] refact --- include/util/encode.h | 14 +++++++------- source/util/src/encode.c | 10 +++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index ee35791012..a3cbc9a42c 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -23,13 +23,6 @@ extern "C" { #endif -typedef struct { - td_endian_t endian; - uint8_t* data; - int32_t size; - int32_t pos; -} SEncoder, SDecoder; - #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) #define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0] @@ -57,6 +50,13 @@ typedef struct { #define tRGet32 tRPut32 #define tRGet64 tRPut64 +typedef struct { + td_endian_t endian; + uint8_t* data; + int32_t size; + int32_t pos; +} SEncoder, SDecoder; + #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) diff --git a/source/util/src/encode.c b/source/util/src/encode.c index 9e648bb281..dc1b926db0 100644 --- a/source/util/src/encode.c +++ b/source/util/src/encode.c @@ -56,8 +56,12 @@ void tCoderClear(SCoder* pCoder) { } int tStartEncode(SCoder* pCoder) { + struct SCoderNode* pNode; + if (pCoder->data) { - struct SCoderNode* pNode = malloc(sizeof(*pNode)); + if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; + + pNode = malloc(sizeof(*pNode)); if (pNode == NULL) return -1; pNode->data = pCoder->data; @@ -97,11 +101,11 @@ int tStartDecode(SCoder* pCoder) { int32_t size; struct SCoderNode* pNode; + // TODO: if (tDecodeI32(pCoder, &size) < 0) return -1; + pNode = malloc(sizeof(*pNode)); if (pNode == NULL) return -1; - // TODO: tDecodeI32(pCoder, &size); - pNode->data = pCoder->data; pNode->pos = pCoder->pos; pNode->size = pCoder->size; From 6420811c8a2f4ff7e483de96758efb82d917e297 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 08:47:28 +0000 Subject: [PATCH 72/97] mor --- include/util/encode.h | 479 ++++++++++++++------------------------- source/util/src/encode.c | 29 +-- 2 files changed, 176 insertions(+), 332 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index a3cbc9a42c..6aa132060b 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -16,6 +16,7 @@ #ifndef _TD_UTIL_ENCODE_H_ #define _TD_UTIL_ENCODE_H_ +#include "freelist.h" #include "tcoding.h" #include "tmacro.h" @@ -50,38 +51,150 @@ extern "C" { #define tRGet32 tRPut32 #define tRGet64 tRPut64 -typedef struct { - td_endian_t endian; - uint8_t* data; - int32_t size; - int32_t pos; -} SEncoder, SDecoder; +typedef enum { TD_ENCODER, TD_DECODER } td_coder_t; +#define CODER_NODE_FIELDS \ + uint8_t* data; \ + int32_t size; \ + int32_t pos; + +struct SCoderNode { + TD_SLIST_NODE(SCoderNode); + CODER_NODE_FIELDS +}; + +typedef struct { + td_coder_t type; + td_endian_t endian; + SFreeList fl; + CODER_NODE_FIELDS + TD_SLIST(SCoderNode) stack; +} SCoder; + +#define TD_CODER_POS(CODER) ((CODER)->pos) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -/* ------------------------ FOR ENCODER ------------------------ */ -static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int32_t size) { - pEncoder->endian = endian; - pEncoder->data = data; - pEncoder->size = (data) ? size : 0; - pEncoder->pos = 0; -} +void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); +void tCoderClear(SCoder* pCoder); + +/* ------------------------ ENCODE ------------------------ */ +int tStartEncode(SCoder* pEncoder); +void tEndEncode(SCoder* pEncoder); +int tEncodeU8(SCoder* pEncoder, uint8_t val); +int tEncodeI8(SCoder* pEncoder, int8_t val); +int tEncodeU16(SCoder* pEncoder, uint16_t val); +int tEncodeI16(SCoder* pEncoder, int16_t val); +int tEncodeU32(SCoder* pEncoder, uint32_t val); +int tEncodeI32(SCoder* pEncoder, int32_t val); +int tEncodeU64(SCoder* pEncoder, uint64_t val); +int tEncodeI64(SCoder* pEncoder, int64_t val); +int tEncodeU16v(SCoder* pEncoder, uint16_t val); +int tEncodeI16v(SCoder* pEncoder, int16_t val); +int tEncodeU32v(SCoder* pEncoder, uint32_t val); +int tEncodeI32v(SCoder* pEncoder, int32_t val); +int tEncodeU64v(SCoder* pEncoder, uint64_t val); +int tEncodeI64v(SCoder* pEncoder, int64_t val); +int tEncodeFloat(SCoder* pEncoder, float val); +int tEncodeDouble(SCoder* pEncoder, double val); +int tEncodeCStr(SCoder* pEncoder, const char* val); + +/* ------------------------ DECODE ------------------------ */ +int tStartDecode(SCoder* pDecoder); +void tEndDecode(SCoder* pDecoder); +int tDecodeU8(SCoder* pDecoder, uint8_t* val); +int tDecodeI8(SCoder* pDecoder, int8_t* val); +int tDecodeU16(SCoder* pDecoder, uint16_t* val); +int tDecodeI16(SCoder* pDecoder, int16_t* val); +int tDecodeU32(SCoder* pDecoder, uint32_t* val); +int tDecodeI32(SCoder* pDecoder, int32_t* val); +int tDecodeU64(SCoder* pDecoder, uint64_t* val); +int tDecodeI64(SCoder* pDecoder, int64_t* val); +int tDecodeU16v(SCoder* pDecoder, uint16_t* val); +int tDecodeI16v(SCoder* pDecoder, int16_t* val); +int tDecodeU32v(SCoder* pDecoder, uint32_t* val); +int tDecodeI32v(SCoder* pDecoder, int32_t* val); +int tDecodeU64v(SCoder* pDecoder, uint64_t* val); +int tDecodeI64v(SCoder* pDecoder, int64_t* val); +int tDecodeFloat(SCoder* pDecoder, float* val); +int tDecodeDouble(SCoder* pDecoder, double* val); +int tDecodeCStr(SCoder* pEncoder, const char** val); + +/* ------------------------ IMPL ------------------------ */ +#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ + if ((CODER)->data) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \ + if (TD_RT_ENDIAN() == (CODER)->endian) { \ + tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \ + } else { \ + tRPut##BITS(TD_CODER_CURRENT(CODER), &(VAL)); \ + } \ + } \ + TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \ + return 0; + +#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \ + while ((VAL) > ENCODE_LIMIT) { \ + if ((CODER)->data) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \ + } \ + \ + (VAL) >>= 7; \ + TD_CODER_MOVE_POS(CODER, 1); \ + } \ + \ + if ((CODER)->data) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \ + } \ + TD_CODER_MOVE_POS(CODER, 1); \ + return 0; + +#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \ + if (TD_RT_ENDIAN() == (CODER)->endian) { \ + tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \ + } else { \ + tRGet##BITS(PVAL, TD_CODER_CURRENT(CODER)); \ + } \ + \ + TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \ + return 0; + +#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \ + int32_t i = 0; \ + *(PVAL) = 0; \ + for (;;) { \ + if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ + TYPE tval = TD_CODER_CURRENT(CODER)[0]; \ + if (tval < ENCODE_LIMIT) { \ + *(PVAL) |= (tval << (7 * i)); \ + TD_CODER_MOVE_POS(pDecoder, 1); \ + break; \ + } else { \ + *(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \ + i++; \ + TD_CODER_MOVE_POS(pDecoder, 1); \ + } \ + } \ + \ + return 0; // 8 -static FORCE_INLINE int tEncodeU8(SEncoder* pEncoder, uint8_t val) { +static FORCE_INLINE int tEncodeU8(SCoder* pEncoder, uint8_t val) { if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; tPut(uint8_t, TD_CODER_CURRENT(pEncoder), val); } TD_CODER_MOVE_POS(pEncoder, sizeof(val)); return 0; } -static FORCE_INLINE int tEncodeI8(SEncoder* pEncoder, int8_t val) { +static FORCE_INLINE int tEncodeI8(SCoder* pEncoder, int8_t val) { if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; + if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; tPut(int8_t, TD_CODER_CURRENT(pEncoder), val); } TD_CODER_MOVE_POS(pEncoder, sizeof(val)); @@ -89,303 +202,77 @@ static FORCE_INLINE int tEncodeI8(SEncoder* pEncoder, int8_t val) { } // 16 -static FORCE_INLINE int tEncodeU16(SEncoder* pEncoder, uint16_t val) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; - if (TD_RT_ENDIAN() == pEncoder->endian) { - tPut(uint16_t, TD_CODER_CURRENT(pEncoder), val); - } else { - tRPut16(TD_CODER_CURRENT(pEncoder), &val); - } - } - TD_CODER_MOVE_POS(pEncoder, sizeof(val)); - return 0; -} - -static FORCE_INLINE int tEncodeI16(SEncoder* pEncoder, int16_t val) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; - if (TD_RT_ENDIAN() == pEncoder->endian) { - tPut(int16_t, TD_CODER_CURRENT(pEncoder), val); - } else { - tRPut16(TD_CODER_CURRENT(pEncoder), &val); - } - } - TD_CODER_MOVE_POS(pEncoder, sizeof(val)); - return 0; -} - +static FORCE_INLINE int tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); } +static FORCE_INLINE int tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); } // 32 -static FORCE_INLINE int tEncodeU32(SEncoder* pEncoder, uint32_t val) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; - if (TD_RT_ENDIAN() == pEncoder->endian) { - tPut(uint32_t, TD_CODER_CURRENT(pEncoder), val); - } else { - tRPut32(TD_CODER_CURRENT(pEncoder), &val); - } - } - TD_CODER_MOVE_POS(pEncoder, sizeof(val)); - return 0; -} - -static FORCE_INLINE int tEncodeI32(SEncoder* pEncoder, int32_t val) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; - if (TD_RT_ENDIAN() == pEncoder->endian) { - tPut(int32_t, TD_CODER_CURRENT(pEncoder), val); - } else { - tRPut32(TD_CODER_CURRENT(pEncoder), &val); - } - } - TD_CODER_MOVE_POS(pEncoder, sizeof(val)); - return 0; -} - +static FORCE_INLINE int tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); } +static FORCE_INLINE int tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); } // 64 -static FORCE_INLINE int tEncodeU64(SEncoder* pEncoder, uint64_t val) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; - if (TD_RT_ENDIAN() == pEncoder->endian) { - tPut(uint64_t, TD_CODER_CURRENT(pEncoder), val); - } else { - tRPut64(TD_CODER_CURRENT(pEncoder), &val); - } - } - TD_CODER_MOVE_POS(pEncoder, sizeof(val)); - return 0; -} - -static FORCE_INLINE int tEncodeI64(SEncoder* pEncoder, int64_t val) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; - if (TD_RT_ENDIAN() == pEncoder->endian) { - tPut(int64_t, TD_CODER_CURRENT(pEncoder), val); - } else { - tRPut64(TD_CODER_CURRENT(pEncoder), &val); - } - } - TD_CODER_MOVE_POS(pEncoder, sizeof(val)); - return 0; -} - +static FORCE_INLINE int tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); } +static FORCE_INLINE int tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); } // 16v -static FORCE_INLINE int tEncodeU16v(SEncoder* pEncoder, uint16_t val) { - int64_t i = 0; - while (val >= ENCODE_LIMIT) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1; - TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff; - } - - val >>= 7; - i++; - } - - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1; - TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val; - } - - TD_CODER_MOVE_POS(pEncoder, i + 1); - - return 0; -} - -static FORCE_INLINE int tEncodeI16v(SEncoder* pEncoder, int16_t val) { +static FORCE_INLINE int tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int tEncodeI16v(SCoder* pEncoder, int16_t val) { return tEncodeU16v(pEncoder, ZIGZAGE(int16_t, val)); } - // 32v -static FORCE_INLINE int tEncodeU32v(SEncoder* pEncoder, uint32_t val) { - int64_t i = 0; - while (val >= ENCODE_LIMIT) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1; - TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff; - } - - val >>= 7; - i++; - } - - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1; - TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val; - } - - TD_CODER_MOVE_POS(pEncoder, i + 1); - - return 0; -} - -static FORCE_INLINE int tEncodeI32v(SEncoder* pEncoder, int32_t val) { +static FORCE_INLINE int tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int tEncodeI32v(SCoder* pEncoder, int32_t val) { return tEncodeU32v(pEncoder, ZIGZAGE(int32_t, val)); } - // 64v -static FORCE_INLINE int tEncodeU64v(SEncoder* pEncoder, uint64_t val) { - int64_t i = 0; - while (val >= ENCODE_LIMIT) { - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1; - TD_CODER_CURRENT(pEncoder)[i] = (val | ENCODE_LIMIT) & 0xff; - } - - val >>= 7; - i++; - } - - if (pEncoder->data) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pEncoder, 1)) return -1; - TD_CODER_CURRENT(pEncoder)[i] = (uint8_t)val; - } - - TD_CODER_MOVE_POS(pEncoder, i + 1); - - return 0; -} - -static FORCE_INLINE int tEncodeI64v(SEncoder* pEncoder, int64_t val) { +static FORCE_INLINE int tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int tEncodeI64v(SCoder* pEncoder, int64_t val) { return tEncodeU64v(pEncoder, ZIGZAGE(int64_t, val)); } -static FORCE_INLINE int tEncodeFloat(SEncoder* pEncoder, float val) { +static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { // TODO return 0; } -static FORCE_INLINE int tEncodeDouble(SEncoder* pEncoder, double val) { +static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { // TODO return 0; } -static FORCE_INLINE int tEncodeCStr(SEncoder* pEncoder, const char* val) { +static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { // TODO return 0; } /* ------------------------ FOR DECODER ------------------------ */ -static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int32_t size) { - ASSERT(!TD_IS_NULL(data)); - pDecoder->endian = endian; - pDecoder->data = data; - pDecoder->size = size; - pDecoder->pos = 0; -} - // 8 -static FORCE_INLINE int tDecodeU8(SDecoder* pDecoder, uint8_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; +static FORCE_INLINE int tDecodeU8(SCoder* pDecoder, uint8_t* val) { + if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; tGet(uint8_t, TD_CODER_CURRENT(pDecoder), *val); TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); return 0; } -static FORCE_INLINE int tDecodeI8(SDecoder* pDecoder, int8_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; +static FORCE_INLINE int tDecodeI8(SCoder* pDecoder, int8_t* val) { + if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; tGet(int8_t, TD_CODER_CURRENT(pDecoder), *val); TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); return 0; } // 16 -static FORCE_INLINE int tDecodeU16(SDecoder* pDecoder, uint16_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; - if (TD_RT_ENDIAN() == pDecoder->endian) { - tGet(uint16_t, TD_CODER_CURRENT(pDecoder), *val); - } else { - tRGet16(val, TD_CODER_CURRENT(pDecoder)); - } - - TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); - return 0; -} - -static FORCE_INLINE int tDecodeI16(SDecoder* pDecoder, int16_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; - if (TD_RT_ENDIAN() == pDecoder->endian) { - tGet(int16_t, TD_CODER_CURRENT(pDecoder), *val); - } else { - tRGet16(val, TD_CODER_CURRENT(pDecoder)); - } - - TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); - return 0; -} - +static FORCE_INLINE int tDecodeU16(SCoder* pDecoder, uint16_t* val) { TD_DECODE_MACRO(pDecoder, val, uint16_t, 16); } +static FORCE_INLINE int tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); } // 32 -static FORCE_INLINE int tDecodeU32(SDecoder* pDecoder, uint32_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; - if (TD_RT_ENDIAN() == pDecoder->endian) { - tGet(uint32_t, TD_CODER_CURRENT(pDecoder), *val); - } else { - tRGet32(val, TD_CODER_CURRENT(pDecoder)); - } - - TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); - return 0; -} - -static FORCE_INLINE int tDecodeI32(SDecoder* pDecoder, int32_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; - if (TD_RT_ENDIAN() == pDecoder->endian) { - tGet(int32_t, TD_CODER_CURRENT(pDecoder), *val); - } else { - tRGet32(val, TD_CODER_CURRENT(pDecoder)); - } - - TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); - return 0; -} - +static FORCE_INLINE int tDecodeU32(SCoder* pDecoder, uint32_t* val) { TD_DECODE_MACRO(pDecoder, val, uint32_t, 32); } +static FORCE_INLINE int tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); } // 64 -static FORCE_INLINE int tDecodeU64(SDecoder* pDecoder, uint64_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; - if (TD_RT_ENDIAN() == pDecoder->endian) { - tGet(uint64_t, TD_CODER_CURRENT(pDecoder), *val); - } else { - tRGet64(val, TD_CODER_CURRENT(pDecoder)); - } - - TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); - return 0; -} - -static FORCE_INLINE int tDecodeI64(SDecoder* pDecoder, int64_t* val) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; - if (TD_RT_ENDIAN() == pDecoder->endian) { - tGet(int64_t, TD_CODER_CURRENT(pDecoder), *val); - } else { - tRGet64(val, TD_CODER_CURRENT(pDecoder)); - } - - TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); - return 0; -} +static FORCE_INLINE int tDecodeU64(SCoder* pDecoder, uint64_t* val) { TD_DECODE_MACRO(pDecoder, val, uint64_t, 64); } +static FORCE_INLINE int tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); } // 16v -static FORCE_INLINE int tDecodeU16v(SDecoder* pDecoder, uint16_t* val) { - int64_t i = 0; - *val = 0; - for (;;) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1; - uint16_t tval = TD_CODER_CURRENT(pDecoder)[i]; - if (tval < ENCODE_LIMIT) { - (*val) |= (tval << (7 * i)); - break; - } else { - (*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); - i++; - } - } - - TD_CODER_MOVE_POS(pDecoder, i); - - return 0; +static FORCE_INLINE int tDecodeU16v(SCoder* pDecoder, uint16_t* val) { + TD_DECODE_VARIANT_MACRO(pDecoder, val, uint16_t); } -static FORCE_INLINE int tDecodeI16v(SDecoder* pDecoder, int16_t* val) { +static FORCE_INLINE int tDecodeI16v(SCoder* pDecoder, int16_t* val) { uint16_t tval; if (tDecodeU16v(pDecoder, &tval) < 0) { return -1; @@ -395,27 +282,11 @@ static FORCE_INLINE int tDecodeI16v(SDecoder* pDecoder, int16_t* val) { } // 32v -static FORCE_INLINE int tDecodeU32v(SDecoder* pDecoder, uint32_t* val) { - int64_t i = 0; - *val = 0; - for (;;) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1; - uint32_t tval = TD_CODER_CURRENT(pDecoder)[i]; - if (tval < ENCODE_LIMIT) { - (*val) |= (tval << (7 * i)); - break; - } else { - (*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); - i++; - } - } - - TD_CODER_MOVE_POS(pDecoder, i); - - return 0; +static FORCE_INLINE int tDecodeU32v(SCoder* pDecoder, uint32_t* val) { + TD_DECODE_VARIANT_MACRO(pDecoder, val, uint32_t); } -static FORCE_INLINE int tDecodeI32v(SDecoder* pDecoder, int32_t* val) { +static FORCE_INLINE int tDecodeI32v(SCoder* pDecoder, int32_t* val) { uint32_t tval; if (tDecodeU32v(pDecoder, &tval) < 0) { return -1; @@ -425,27 +296,11 @@ static FORCE_INLINE int tDecodeI32v(SDecoder* pDecoder, int32_t* val) { } // 64v -static FORCE_INLINE int tDecodeU64v(SDecoder* pDecoder, uint64_t* val) { - int64_t i = 0; - *val = 0; - for (;;) { - if (TD_CHECK_CODER_CAPACITY_FAILED(pDecoder, 1)) return -1; - uint64_t tval = TD_CODER_CURRENT(pDecoder)[i]; - if (tval < ENCODE_LIMIT) { - (*val) |= (tval << (7 * i)); - break; - } else { - (*val) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); - i++; - } - } - - TD_CODER_MOVE_POS(pDecoder, i); - - return 0; +static FORCE_INLINE int tDecodeU64v(SCoder* pDecoder, uint64_t* val) { + TD_DECODE_VARIANT_MACRO(pDecoder, val, uint64_t); } -static FORCE_INLINE int tDecodeI64v(SDecoder* pDecoder, int64_t* val) { +static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) { uint64_t tval; if (tDecodeU64v(pDecoder, &tval) < 0) { return -1; @@ -454,17 +309,17 @@ static FORCE_INLINE int tDecodeI64v(SDecoder* pDecoder, int64_t* val) { return 0; } -static FORCE_INLINE int tDecodeFloat(SDecoder* pDecoder, float* val) { +static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) { // TODO return 0; } -static FORCE_INLINE int tDecodeDouble(SDecoder* pDecoder, double* val) { +static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { // TODO return 0; } -static FORCE_INLINE int tDecodeCStr(SDecoder* pEncoder, const char** val) { +static FORCE_INLINE int tDecodeCStr(SCoder* pEncoder, const char** val) { // TODO return 0; } diff --git a/source/util/src/encode.c b/source/util/src/encode.c index dc1b926db0..192ecefe43 100644 --- a/source/util/src/encode.c +++ b/source/util/src/encode.c @@ -14,28 +14,15 @@ */ #include "encode.h" -#include "freelist.h" -#define CODER_NODE_FIELDS \ - uint8_t* data; \ - int32_t size; \ - int32_t pos; +void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type) { + if (type == TD_ENCODER) { + if (data == NULL) size = 0; + } else { + ASSERT(data && size > 0); + } -struct SCoderNode { - TD_SLIST_NODE(SCoderNode); - CODER_NODE_FIELDS -}; - -typedef struct { - td_endian_t endian; - SFreeList fl; - CODER_NODE_FIELDS - TD_SLIST(SCoderNode) stack; -} SCoder; - -bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } - -void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size) { + pCoder->type = type; pCoder->endian = endian; pCoder->data = data; pCoder->size = size; @@ -55,6 +42,8 @@ void tCoderClear(SCoder* pCoder) { } } +bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } + int tStartEncode(SCoder* pCoder) { struct SCoderNode* pNode; From 6b6f3f75af8240c8120ca0b902b87952091ae2ba Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 09:27:33 +0000 Subject: [PATCH 73/97] refact --- include/common/tmsg.h | 6 ++- include/util/encode.h | 78 ++++++++++++++-------------- source/client/CMakeLists.txt | 4 +- source/common/CMakeLists.txt | 4 +- source/common/src/tmsg.c | 10 +++- source/libs/catalog/CMakeLists.txt | 4 +- source/libs/parser/CMakeLists.txt | 4 +- source/libs/planner/CMakeLists.txt | 4 +- source/libs/qcom/CMakeLists.txt | 8 +-- source/libs/qworker/CMakeLists.txt | 4 +- source/libs/scheduler/CMakeLists.txt | 4 +- source/util/CMakeLists.txt | 5 +- 12 files changed, 82 insertions(+), 53 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index af250bff03..5365a0ee36 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -58,6 +58,7 @@ extern int tMsgDict[]; typedef uint16_t tmsg_t; /* ------------------------ ENCODE/DECODE FUNCTIONS AND MACROS ------------------------ */ +#if 0 struct SMEListNode { TD_SLIST_NODE(SMEListNode); SEncoder coder; @@ -96,6 +97,7 @@ typedef struct SMsgDecoder { void tmsgInitMsgDecoder(SMsgDecoder* pMD, td_endian_t endian, uint8_t* data, int64_t size); void tmsgClearMsgDecoder(SMsgDecoder* pMD); +#endif /* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type @@ -1283,8 +1285,8 @@ typedef struct { SArray* pArray; } SVCreateTbBatchReq; -int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); -int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); +// int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); +// int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); int tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); int tSVCreateTbBatchReqSerialize(void** buf, SVCreateTbBatchReq* pReq); diff --git a/include/util/encode.h b/include/util/encode.h index 6aa132060b..5c7d3134ae 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -74,52 +74,52 @@ typedef struct { #define TD_CODER_POS(CODER) ((CODER)->pos) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) -#define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) +#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); /* ------------------------ ENCODE ------------------------ */ -int tStartEncode(SCoder* pEncoder); -void tEndEncode(SCoder* pEncoder); -int tEncodeU8(SCoder* pEncoder, uint8_t val); -int tEncodeI8(SCoder* pEncoder, int8_t val); -int tEncodeU16(SCoder* pEncoder, uint16_t val); -int tEncodeI16(SCoder* pEncoder, int16_t val); -int tEncodeU32(SCoder* pEncoder, uint32_t val); -int tEncodeI32(SCoder* pEncoder, int32_t val); -int tEncodeU64(SCoder* pEncoder, uint64_t val); -int tEncodeI64(SCoder* pEncoder, int64_t val); -int tEncodeU16v(SCoder* pEncoder, uint16_t val); -int tEncodeI16v(SCoder* pEncoder, int16_t val); -int tEncodeU32v(SCoder* pEncoder, uint32_t val); -int tEncodeI32v(SCoder* pEncoder, int32_t val); -int tEncodeU64v(SCoder* pEncoder, uint64_t val); -int tEncodeI64v(SCoder* pEncoder, int64_t val); -int tEncodeFloat(SCoder* pEncoder, float val); -int tEncodeDouble(SCoder* pEncoder, double val); -int tEncodeCStr(SCoder* pEncoder, const char* val); +static int tStartEncode(SCoder* pEncoder); +static void tEndEncode(SCoder* pEncoder); +static int tEncodeU8(SCoder* pEncoder, uint8_t val); +static int tEncodeI8(SCoder* pEncoder, int8_t val); +static int tEncodeU16(SCoder* pEncoder, uint16_t val); +static int tEncodeI16(SCoder* pEncoder, int16_t val); +static int tEncodeU32(SCoder* pEncoder, uint32_t val); +static int tEncodeI32(SCoder* pEncoder, int32_t val); +static int tEncodeU64(SCoder* pEncoder, uint64_t val); +static int tEncodeI64(SCoder* pEncoder, int64_t val); +static int tEncodeU16v(SCoder* pEncoder, uint16_t val); +static int tEncodeI16v(SCoder* pEncoder, int16_t val); +static int tEncodeU32v(SCoder* pEncoder, uint32_t val); +static int tEncodeI32v(SCoder* pEncoder, int32_t val); +static int tEncodeU64v(SCoder* pEncoder, uint64_t val); +static int tEncodeI64v(SCoder* pEncoder, int64_t val); +static int tEncodeFloat(SCoder* pEncoder, float val); +static int tEncodeDouble(SCoder* pEncoder, double val); +static int tEncodeCStr(SCoder* pEncoder, const char* val); /* ------------------------ DECODE ------------------------ */ -int tStartDecode(SCoder* pDecoder); -void tEndDecode(SCoder* pDecoder); -int tDecodeU8(SCoder* pDecoder, uint8_t* val); -int tDecodeI8(SCoder* pDecoder, int8_t* val); -int tDecodeU16(SCoder* pDecoder, uint16_t* val); -int tDecodeI16(SCoder* pDecoder, int16_t* val); -int tDecodeU32(SCoder* pDecoder, uint32_t* val); -int tDecodeI32(SCoder* pDecoder, int32_t* val); -int tDecodeU64(SCoder* pDecoder, uint64_t* val); -int tDecodeI64(SCoder* pDecoder, int64_t* val); -int tDecodeU16v(SCoder* pDecoder, uint16_t* val); -int tDecodeI16v(SCoder* pDecoder, int16_t* val); -int tDecodeU32v(SCoder* pDecoder, uint32_t* val); -int tDecodeI32v(SCoder* pDecoder, int32_t* val); -int tDecodeU64v(SCoder* pDecoder, uint64_t* val); -int tDecodeI64v(SCoder* pDecoder, int64_t* val); -int tDecodeFloat(SCoder* pDecoder, float* val); -int tDecodeDouble(SCoder* pDecoder, double* val); -int tDecodeCStr(SCoder* pEncoder, const char** val); +static int tStartDecode(SCoder* pDecoder); +static void tEndDecode(SCoder* pDecoder); +static int tDecodeU8(SCoder* pDecoder, uint8_t* val); +static int tDecodeI8(SCoder* pDecoder, int8_t* val); +static int tDecodeU16(SCoder* pDecoder, uint16_t* val); +static int tDecodeI16(SCoder* pDecoder, int16_t* val); +static int tDecodeU32(SCoder* pDecoder, uint32_t* val); +static int tDecodeI32(SCoder* pDecoder, int32_t* val); +static int tDecodeU64(SCoder* pDecoder, uint64_t* val); +static int tDecodeI64(SCoder* pDecoder, int64_t* val); +static int tDecodeU16v(SCoder* pDecoder, uint16_t* val); +static int tDecodeI16v(SCoder* pDecoder, int16_t* val); +static int tDecodeU32v(SCoder* pDecoder, uint32_t* val); +static int tDecodeI32v(SCoder* pDecoder, int32_t* val); +static int tDecodeU64v(SCoder* pDecoder, uint64_t* val); +static int tDecodeI64v(SCoder* pDecoder, int64_t* val); +static int tDecodeFloat(SCoder* pDecoder, float* val); +static int tDecodeDouble(SCoder* pDecoder, double* val); +static int tDecodeCStr(SCoder* pEncoder, const char** val); /* ------------------------ IMPL ------------------------ */ #define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index ad6a93415c..3210c0c6de 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -11,4 +11,6 @@ target_link_libraries( PRIVATE os util common transport parser planner catalog scheduler function qcom ) -ADD_SUBDIRECTORY(test) \ No newline at end of file +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 1ff83b091d..04e5a36e86 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -12,4 +12,6 @@ target_link_libraries( INTERFACE api ) -ADD_SUBDIRECTORY(test) +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a18a472dba..c0baa1e411 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,12 +27,15 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +#if 0 static int tmsgStartEncode(SMsgEncoder *pME); static void tmsgEndEncode(SMsgEncoder *pME); static int tmsgStartDecode(SMsgDecoder *pMD); static void tmsgEndDecode(SMsgDecoder *pMD); +#endif /* ------------------------ ENCODE/DECODE FUNCTIONS ------------------------ */ +#if 0 void tmsgInitMsgEncoder(SMsgEncoder *pME, td_endian_t endian, uint8_t *data, int64_t size) { tInitEncoder(&(pME->coder), endian, data, size); TD_SLIST_INIT(&(pME->eStack)); @@ -74,8 +77,10 @@ void tmsgClearMsgDecoder(SMsgDecoder *pMD) { } } } +#endif /* ------------------------ MESSAGE ENCODE/DECODE ------------------------ */ +#if 0 int tmsgSVCreateTbReqEncode(SMsgEncoder *pCoder, SVCreateTbReq *pReq) { tmsgStartEncode(pCoder); // TODO @@ -97,6 +102,7 @@ int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) { tmsgEndDecode(pCoder); return 0; } +#endif int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int tlen = 0; @@ -221,6 +227,7 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { } /* ------------------------ STATIC METHODS ------------------------ */ +#if 0 static int tmsgStartEncode(SMsgEncoder *pME) { struct SMEListNode *pNode = (struct SMEListNode *)malloc(sizeof(*pNode)); if (TD_IS_NULL(pNode)) return -1; @@ -278,4 +285,5 @@ static void tmsgEndDecode(SMsgDecoder *pMD) { pMD->coder = pNode->coder; free(pNode); -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index f47e105b8a..bb9ed686a7 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -11,4 +11,6 @@ target_link_libraries( PRIVATE os util transport qcom ) -ADD_SUBDIRECTORY(test) \ No newline at end of file +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 6ab3020935..7f77876c4c 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -11,4 +11,6 @@ target_link_libraries( PRIVATE os util catalog function transport qcom ) -ADD_SUBDIRECTORY(test) +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 7f8c118663..6234dbe0ac 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -11,4 +11,6 @@ target_link_libraries( PRIVATE os util catalog cjson parser transport function qcom ) -ADD_SUBDIRECTORY(test) +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index 8e09f3d97a..c63e54fb9b 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -7,8 +7,10 @@ target_include_directories( ) target_link_libraries( - qcom - PRIVATE os util transport + qcom + PRIVATE os util transport ) -ADD_SUBDIRECTORY(test) +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index 001756d7c3..ea3e97057b 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -11,4 +11,6 @@ target_link_libraries( PRIVATE os util transport planner qcom ) -ADD_SUBDIRECTORY(test) \ No newline at end of file +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 6baaab1ef4..4e297f4e17 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -12,4 +12,6 @@ target_link_libraries( PRIVATE os util planner qcom common catalog transport ) -ADD_SUBDIRECTORY(test) \ No newline at end of file +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index bf1774b45b..ed936e90f6 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -14,4 +14,7 @@ target_link_libraries( PUBLIC api ) -ADD_SUBDIRECTORY(test) +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) + From 6d3032d538c82715784eb0c1dd80dc96faa242f6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 09:40:11 +0000 Subject: [PATCH 74/97] more --- include/util/encode.h | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index 5c7d3134ae..4164a7660f 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -227,13 +227,21 @@ static FORCE_INLINE int tEncodeI64v(SCoder* pEncoder, int64_t val) { } static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { - // TODO - return 0; + union { + uint32_t ui; + float f; + } v = {.f = val}; + + return tEncodeU32(pEncoder, v.ui); } static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { - // TODO - return 0; + union { + uint64_t ui; + double d; + } v = {.d = val}; + + return tEncodeU64(pEncoder, v.ui); } static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { @@ -310,12 +318,30 @@ static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) { } static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) { - // TODO + union { + uint32_t ui; + float f; + } v; + + if (tDecodeU32(pDecoder, &(v.ui)) < 0) { + return -1; + } + + *val = v.f; return 0; } static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { - // TODO + union { + uint64_t ui; + double d; + } v; + + if (tDecodeU64(pEncoder, &(v.ui)) < 0) { + return -1; + } + + *val = v.d; return 0; } From e4252065d1a4d113ae66baace85765433e2085dd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 09:41:00 +0000 Subject: [PATCH 75/97] refact --- include/util/encode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/encode.h b/include/util/encode.h index 4164a7660f..3c98e8108f 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -236,7 +236,7 @@ static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { } static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { - union { + union { uint64_t ui; double d; } v = {.d = val}; From ec03a0eb8fdb7059fad42bac3ee991ec88d36e44 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 09:42:57 +0000 Subject: [PATCH 76/97] more --- source/util/src/encode.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/util/src/encode.c b/source/util/src/encode.c index 192ecefe43..a3946daf5c 100644 --- a/source/util/src/encode.c +++ b/source/util/src/encode.c @@ -15,6 +15,11 @@ #include "encode.h" +#if __STDC_VERSION__ >= 201112L +static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); +static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)"); +#endif + void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type) { if (type == TD_ENCODER) { if (data == NULL) size = 0; From 40fc4b6cf38c0373de43be0f68638fd858c1a562 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 2 Jan 2022 17:50:40 +0800 Subject: [PATCH 77/97] fix mem leak and invalid read/write found by valgrind --- source/libs/index/inc/index_tfile.h | 1 + source/libs/index/src/index.c | 3 ++- source/libs/index/src/index_cache.c | 1 + source/libs/index/src/index_fst.c | 2 ++ source/libs/index/src/index_tfile.c | 9 ++++++--- source/libs/index/test/indexTests.cc | 29 +++++++++++++++++++++++++--- 6 files changed, 38 insertions(+), 7 deletions(-) diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 675203fa1a..4618a39197 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -117,6 +117,7 @@ int tfileWriterFinish(TFileWriter* tw); // IndexTFile* indexTFileCreate(const char* path); +void indexTFileDestroy(IndexTFile* tfile); int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 11a2aa5e5f..9c7320b301 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -103,6 +103,7 @@ void indexClose(SIndex* sIdx) { } taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); + indexTFileDestroy(sIdx->tindex); #endif free(sIdx->path); free(sIdx); @@ -479,7 +480,7 @@ void iterateValueDestroy(IterateValue* value, bool destroy) { } else { if (value->val != NULL) { taosArrayClear(value->val); } } - // free(value->colVal); + free(value->colVal); value->colVal = NULL; } static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 0f00d9d4af..b4c533e998 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -150,6 +150,7 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { MemTable* tbl = cache->imm; iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); + iiter->val.colVal = NULL; iiter->iter = tbl != NULL ? tSkipListCreateIter(tbl->mem) : NULL; iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 088c7369d5..bfaeeaaa33 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1062,6 +1062,7 @@ Output fstEmptyFinalOutput(Fst* fst, bool* null) { } else { *null = true; } + fstNodeDestroy(node); return res; } @@ -1286,6 +1287,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); free(buf); fstSliceDestroy(&slice); + taosArrayDestroy(nodes); return result; } free(buf); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index fe54b5ebb3..95c713fb0a 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -360,7 +360,7 @@ IndexTFile* indexTFileCreate(const char* path) { tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile* tfile) { +void indexTFileDestroy(IndexTFile* tfile) { tfileCacheDestroy(tfile->cache); free(tfile); } @@ -415,7 +415,7 @@ static bool tfileIteratorNext(Iterate* iiter) { static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { - TFileFstIter* tIter = calloc(1, sizeof(Iterate)); + TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); if (tIter == NULL) { return NULL; } tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); @@ -437,6 +437,7 @@ Iterate* tfileIteratorCreate(TFileReader* reader) { iter->next = tfileIteratorNext; iter->getValue = tifileIterateGetValue; iter->val.val = taosArrayInit(1, sizeof(uint64_t)); + iter->val.colVal = NULL; return iter; } void tfileIteratorDestroy(Iterate* iter) { @@ -449,6 +450,7 @@ void tfileIteratorDestroy(Iterate* iter) { streamWithStateDestroy(tIter->st); fstStreamBuilderDestroy(tIter->fb); automCtxDestroy(tIter->ctx); + free(tIter); free(iter); } @@ -482,7 +484,7 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); if (tf == NULL) { return NULL; } - tf->colVal = val; + tf->colVal = tstrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } @@ -493,6 +495,7 @@ int tfileValuePush(TFileValue* tf, uint64_t val) { } void tfileValueDestroy(TFileValue* tf) { taosArrayDestroy(tf->tableId); + free(tf->colVal); free(tf); } static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 7bf40bc7b3..bdfb86ce17 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -457,7 +457,10 @@ TEST_F(IndexTFileEnv, test_tfile_write) { // taosArrayPush(data, &v4); fObj->Put(data); - for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); } + for (size_t i = 0; i < taosArrayGetSize(data); i++) { + // data + destroyTFileValue(taosArrayGetP(data, i)); + } taosArrayDestroy(data); std::string colName("voltage"); @@ -470,6 +473,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { fObj->Get(&query, result); assert(taosArrayGetSize(result) == 200); indexTermDestroy(term); + taosArrayDestroy(result); // tfileWriterDestroy(twrite); } @@ -534,6 +538,7 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); // indexTermDestry(term); } { @@ -541,24 +546,28 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v2"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } coj->Debug(); std::cout << "--------first----------" << std::endl; @@ -567,12 +576,14 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); + indexTermDestroy(term); } { std::string colVal("v4"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); + indexTermDestroy(term); } coj->Debug(); std::cout << "--------second----------" << std::endl; @@ -583,6 +594,7 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + indexTermDestroy(term); } } coj->Debug(); @@ -598,6 +610,9 @@ TEST_F(IndexCacheEnv, cache_test) { coj->Get(&query, colId, 10000, ret, &valType); std::cout << "size : " << taosArrayGetSize(ret) << std::endl; assert(taosArrayGetSize(ret) == 4); + taosArrayDestroy(ret); + + indexTermDestroy(term); } { std::string colVal("v2"); @@ -609,6 +624,9 @@ TEST_F(IndexCacheEnv, cache_test) { coj->Get(&query, colId, 10000, ret, &valType); assert(taosArrayGetSize(ret) == 1); + taosArrayDestroy(ret); + + indexTermDestroy(term); } } class IndexObj { @@ -678,13 +696,16 @@ class IndexObj { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; } - return taosArrayGetSize(result); + int sz = taosArrayGetSize(result); + indexMultiTermQueryDestroy(mq); + taosArrayDestroy(result); + return sz; // assert(taosArrayGetSize(result) == targetSize); } void PutOne(const std::string& colName, const std::string& colVal) { + SIndexMultiTerm* terms = indexMultiTermCreate(); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); Put(terms, 10); indexMultiTermDestroy(terms); @@ -783,6 +804,8 @@ TEST_F(IndexEnv2, testIndexOpen) { index->Search(mq, result); std::cout << "target size: " << taosArrayGetSize(result) << std::endl; assert(taosArrayGetSize(result) == 400); + taosArrayDestroy(result); + indexMultiTermQueryDestroy(mq); } } From 82be83a22616c8868231ca02eb8489d4bd3f9acf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 09:51:47 +0000 Subject: [PATCH 78/97] more --- include/util/encode.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index 3c98e8108f..7dc8d07f0a 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -244,11 +244,20 @@ static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { return tEncodeU64(pEncoder, v.ui); } -static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { - // TODO +static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, size_t len) { + if (tEncodeI32v(pEncoder, len) < 0) return -1; + if (pEncoder->data) { + if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len + 1)) return -1; + memcpy(TD_CODER_CURRENT(pEncoder), val, len + 1); + } + TD_CODER_MOVE_POS(pEncoder, len + 1); return 0; } +static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { + return tEncodeCstrWithLen(pEncoder, val, strlen(val)); +} + /* ------------------------ FOR DECODER ------------------------ */ // 8 static FORCE_INLINE int tDecodeU8(SCoder* pDecoder, uint8_t* val) { From 6835e4956be283f2bb1184772a96e284f919cac7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:03:21 +0000 Subject: [PATCH 79/97] more --- include/util/encode.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index 7dc8d07f0a..0b90d66f5a 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -98,6 +98,7 @@ static int tEncodeU64v(SCoder* pEncoder, uint64_t val); static int tEncodeI64v(SCoder* pEncoder, int64_t val); static int tEncodeFloat(SCoder* pEncoder, float val); static int tEncodeDouble(SCoder* pEncoder, double val); +static int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len); static int tEncodeCStr(SCoder* pEncoder, const char* val); /* ------------------------ DECODE ------------------------ */ @@ -119,6 +120,7 @@ static int tDecodeU64v(SCoder* pDecoder, uint64_t* val); static int tDecodeI64v(SCoder* pDecoder, int64_t* val); static int tDecodeFloat(SCoder* pDecoder, float* val); static int tDecodeDouble(SCoder* pDecoder, double* val); +static int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); static int tDecodeCStr(SCoder* pEncoder, const char** val); /* ------------------------ IMPL ------------------------ */ @@ -244,8 +246,8 @@ static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { return tEncodeU64(pEncoder, v.ui); } -static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, size_t len) { - if (tEncodeI32v(pEncoder, len) < 0) return -1; +static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { + if (tEncodeU64v(pEncoder, len) < 0) return -1; if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len + 1)) return -1; memcpy(TD_CODER_CURRENT(pEncoder), val, len + 1); @@ -255,7 +257,7 @@ static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, si } static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { - return tEncodeCstrWithLen(pEncoder, val, strlen(val)); + return tEncodeCstrWithLen(pEncoder, val, (uint64_t)strlen(val)); } /* ------------------------ FOR DECODER ------------------------ */ @@ -346,7 +348,7 @@ static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { double d; } v; - if (tDecodeU64(pEncoder, &(v.ui)) < 0) { + if (tDecodeU64(pDecoder, &(v.ui)) < 0) { return -1; } @@ -354,11 +356,21 @@ static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { return 0; } -static FORCE_INLINE int tDecodeCStr(SCoder* pEncoder, const char** val) { - // TODO +static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { + if (tDecodeU64v(pDecoder, len) < 0) return -1; + + if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len + 1)) return -1; + *val = (char *)TD_CODER_CURRENT(pDecoder); + + TD_CODER_MOVE_POS(pDecoder, *len + 1); return 0; } +static FORCE_INLINE int tDecodeCStr(SCoder* pDecoder, const char** val) { + uint64_t len; + return tDecodeCStrAndLen(pDecoder, val, &len); +} + #ifdef __cplusplus } #endif From 17852c2c431f558cae31182f9b5af5555661013f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:05:05 +0000 Subject: [PATCH 80/97] more --- include/util/encode.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/util/encode.h b/include/util/encode.h index 0b90d66f5a..b3ebda4ed7 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -75,6 +75,7 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) +#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); From bac6f05b96a9998b8e5ef7d8b4228f7b56c4105f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:06:00 +0000 Subject: [PATCH 81/97] refact --- include/util/encode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/encode.h b/include/util/encode.h index b3ebda4ed7..7585566dc9 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -361,7 +361,7 @@ static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, ui if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len + 1)) return -1; - *val = (char *)TD_CODER_CURRENT(pDecoder); + *val = (char*)TD_CODER_CURRENT(pDecoder); TD_CODER_MOVE_POS(pDecoder, *len + 1); return 0; From ecaf4268987976a679ac4ba9ac1e7d60c168f2a3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:15:52 +0000 Subject: [PATCH 82/97] more --- include/util/encode.h | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/include/util/encode.h b/include/util/encode.h index 7585566dc9..c916d8fc6a 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -99,6 +99,7 @@ static int tEncodeU64v(SCoder* pEncoder, uint64_t val); static int tEncodeI64v(SCoder* pEncoder, int64_t val); static int tEncodeFloat(SCoder* pEncoder, float val); static int tEncodeDouble(SCoder* pEncoder, double val); +static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); static int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len); static int tEncodeCStr(SCoder* pEncoder, const char* val); @@ -121,8 +122,9 @@ static int tDecodeU64v(SCoder* pDecoder, uint64_t* val); static int tDecodeI64v(SCoder* pDecoder, int64_t* val); static int tDecodeFloat(SCoder* pDecoder, float* val); static int tDecodeDouble(SCoder* pDecoder, double* val); +static int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len); static int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); -static int tDecodeCStr(SCoder* pEncoder, const char** val); +static int tDecodeCStr(SCoder* pDecoder, const char** val); /* ------------------------ IMPL ------------------------ */ #define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ @@ -247,6 +249,17 @@ static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { return tEncodeU64(pEncoder, v.ui); } +static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) { + if (tEncodeU64v(pEncoder, len) < 0) return -1; + if (pEncoder->data) { + if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len)) return -1; + memcpy(TD_CODER_CURRENT(pEncoder), len); + } + + TD_CODER_MOVE_POS(pEncoder, len); + return 0; +} + static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { if (tEncodeU64v(pEncoder, len) < 0) return -1; if (pEncoder->data) { @@ -357,6 +370,16 @@ static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { return 0; } +static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) { + if (tDecodeU64v(pDecoder, &len) < 0) return -1; + + if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; + *val = (void*)TD_CODER_CURRENT(pDecoder); + + TD_CODER_MOVE_POS(pDecoder, *len); + return 0; +} + static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { if (tDecodeU64v(pDecoder, len) < 0) return -1; From da8e603c49f9091780e0aa5fda324ebbf535b64b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:21:37 +0000 Subject: [PATCH 83/97] more --- include/util/encode.h | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index c916d8fc6a..f5c8db0b5a 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -253,7 +253,7 @@ static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_ if (tEncodeU64v(pEncoder, len) < 0) return -1; if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len)) return -1; - memcpy(TD_CODER_CURRENT(pEncoder), len); + memcpy(TD_CODER_CURRENT(pEncoder), val, len); } TD_CODER_MOVE_POS(pEncoder, len); @@ -261,13 +261,7 @@ static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_ } static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { - if (tEncodeU64v(pEncoder, len) < 0) return -1; - if (pEncoder->data) { - if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len + 1)) return -1; - memcpy(TD_CODER_CURRENT(pEncoder), val, len + 1); - } - TD_CODER_MOVE_POS(pEncoder, len + 1); - return 0; + return tEncodeBinary(pEncoder, (void*)val, len + 1); } static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { @@ -371,7 +365,7 @@ static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { } static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) { - if (tDecodeU64v(pDecoder, &len) < 0) return -1; + if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; *val = (void*)TD_CODER_CURRENT(pDecoder); @@ -381,12 +375,8 @@ static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64 } static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { - if (tDecodeU64v(pDecoder, len) < 0) return -1; - - if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len + 1)) return -1; - *val = (char*)TD_CODER_CURRENT(pDecoder); - - TD_CODER_MOVE_POS(pDecoder, *len + 1); + if (tDecodeBinary(pDecoder, (const void**)val, len) < 0) return -1; + (*len) -= 1; return 0; } From 8f1a1b7bbc5b5d15ea49d9e8e5944c518e71984c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:26:32 +0000 Subject: [PATCH 84/97] refact --- include/common/tmsg.h | 42 ------------ source/common/src/tmsg.c | 141 +-------------------------------------- 2 files changed, 1 insertion(+), 182 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5365a0ee36..6c2b4a1acc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -57,48 +57,6 @@ extern int tMsgDict[]; typedef uint16_t tmsg_t; -/* ------------------------ ENCODE/DECODE FUNCTIONS AND MACROS ------------------------ */ -#if 0 -struct SMEListNode { - TD_SLIST_NODE(SMEListNode); - SEncoder coder; -}; - -typedef struct SMsgEncoder { - SEncoder coder; - TD_SLIST(SMEListNode) eStack; // encode stack -} SMsgEncoder; - -struct SMDFreeListNode { - TD_SLIST_NODE(SMDFreeListNode); - char payload[]; -}; - -struct SMDListNode { - TD_SLIST_NODE(SMDListNode); - SDecoder coder; -}; - -typedef struct SMsgDecoder { - SDecoder coder; - TD_SLIST(SMDListNode) dStack; - TD_SLIST(SMDFreeListNode) freeList; -} SMsgDecoder; - -#define TMSG_MALLOC(SIZE, DECODER) \ - ({ \ - void* ptr = malloc((SIZE) + sizeof(struct SMDFreeListNode)); \ - if (ptr) { \ - TD_SLIST_PUSH(&((DECODER)->freeList), (struct SMDFreeListNode*)ptr); \ - ptr = POINTER_SHIFT(ptr, sizeof(struct SMDFreeListNode*)); \ - } \ - ptr; \ - }) - -void tmsgInitMsgDecoder(SMsgDecoder* pMD, td_endian_t endian, uint8_t* data, int64_t size); -void tmsgClearMsgDecoder(SMsgDecoder* pMD); -#endif - /* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c0baa1e411..0874c471aa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,83 +27,6 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -#if 0 -static int tmsgStartEncode(SMsgEncoder *pME); -static void tmsgEndEncode(SMsgEncoder *pME); -static int tmsgStartDecode(SMsgDecoder *pMD); -static void tmsgEndDecode(SMsgDecoder *pMD); -#endif - -/* ------------------------ ENCODE/DECODE FUNCTIONS ------------------------ */ -#if 0 -void tmsgInitMsgEncoder(SMsgEncoder *pME, td_endian_t endian, uint8_t *data, int64_t size) { - tInitEncoder(&(pME->coder), endian, data, size); - TD_SLIST_INIT(&(pME->eStack)); -} - -void tmsgClearMsgEncoder(SMsgEncoder *pME) { - struct SMEListNode *pNode; - for (;;) { - pNode = TD_SLIST_HEAD(&(pME->eStack)); - if (TD_IS_NULL(pNode)) break; - TD_SLIST_POP(&(pME->eStack)); - free(pNode); - } -} - -void tmsgInitMsgDecoder(SMsgDecoder *pMD, td_endian_t endian, uint8_t *data, int64_t size) { - tInitDecoder(&pMD->coder, endian, data, size); - TD_SLIST_INIT(&(pMD->dStack)); - TD_SLIST_INIT(&(pMD->freeList)); -} - -void tmsgClearMsgDecoder(SMsgDecoder *pMD) { - { - struct SMDFreeListNode *pNode; - for (;;) { - pNode = TD_SLIST_HEAD(&(pMD->freeList)); - if (TD_IS_NULL(pNode)) break; - TD_SLIST_POP(&(pMD->freeList)); - free(pNode); - } - } - { - struct SMDListNode *pNode; - for (;;) { - pNode = TD_SLIST_HEAD(&(pMD->dStack)); - if (TD_IS_NULL(pNode)) break; - TD_SLIST_POP(&(pMD->dStack)); - free(pNode); - } - } -} -#endif - -/* ------------------------ MESSAGE ENCODE/DECODE ------------------------ */ -#if 0 -int tmsgSVCreateTbReqEncode(SMsgEncoder *pCoder, SVCreateTbReq *pReq) { - tmsgStartEncode(pCoder); - // TODO - - tmsgEndEncode(pCoder); - return 0; -} - -int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) { - tmsgStartDecode(pCoder); - - // TODO: decode - - // Decode is not end - if (pCoder->coder.pos != pCoder->coder.size) { - // Continue decode - } - - tmsgEndDecode(pCoder); - return 0; -} -#endif - int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int tlen = 0; @@ -224,66 +147,4 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { } return buf; -} - -/* ------------------------ STATIC METHODS ------------------------ */ -#if 0 -static int tmsgStartEncode(SMsgEncoder *pME) { - struct SMEListNode *pNode = (struct SMEListNode *)malloc(sizeof(*pNode)); - if (TD_IS_NULL(pNode)) return -1; - - pNode->coder = pME->coder; - TD_SLIST_PUSH(&(pME->eStack), pNode); - TD_CODER_MOVE_POS(&(pME->coder), sizeof(int32_t)); - - return 0; -} - -static void tmsgEndEncode(SMsgEncoder *pME) { - int32_t size; - struct SMEListNode *pNode; - - pNode = TD_SLIST_HEAD(&(pME->eStack)); - ASSERT(pNode); - TD_SLIST_POP(&(pME->eStack)); - - size = pME->coder.pos - pNode->coder.pos; - tEncodeI32(&(pNode->coder), size); - - free(pNode); -} - -static int tmsgStartDecode(SMsgDecoder *pMD) { - struct SMDListNode *pNode; - int32_t size; - - pNode = (struct SMDListNode *)malloc(sizeof(*pNode)); - if (pNode == NULL) return -1; - - tDecodeI32(&(pMD->coder), &size); - - pNode->coder = pMD->coder; - TD_SLIST_PUSH(&(pMD->dStack), pNode); - - pMD->coder.pos = 0; - pMD->coder.size = size - sizeof(int32_t); - pMD->coder.data = TD_CODER_CURRENT(&(pNode->coder)); - - return 0; -} - -static void tmsgEndDecode(SMsgDecoder *pMD) { - ASSERT(pMD->coder.pos == pMD->coder.size); - struct SMDListNode *pNode; - - pNode = TD_SLIST_HEAD(&(pMD->dStack)); - ASSERT(pNode); - TD_SLIST_POP(&(pMD->dStack)); - - pNode->coder.pos += pMD->coder.size; - - pMD->coder = pNode->coder; - - free(pNode); -} -#endif \ No newline at end of file +} \ No newline at end of file From 729aabca1591540002121139c3d002276f077ef4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:29:47 +0000 Subject: [PATCH 85/97] refact --- include/common/tmsg.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6c2b4a1acc..85e75139ce 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1243,8 +1243,6 @@ typedef struct { SArray* pArray; } SVCreateTbBatchReq; -// int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); -// int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); int tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); int tSVCreateTbBatchReqSerialize(void** buf, SVCreateTbBatchReq* pReq); From eaca069a45b8b9ff4b98245f91a58edfeca5d16c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 2 Jan 2022 10:34:31 +0000 Subject: [PATCH 86/97] more --- include/util/encode.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/util/encode.h b/include/util/encode.h index f5c8db0b5a..02edbe888e 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -125,6 +125,7 @@ static int tDecodeDouble(SCoder* pDecoder, double* val); static int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len); static int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); static int tDecodeCStr(SCoder* pDecoder, const char** val); +static int tDecodeCStrTo(SCoder* pDecoder, char* val); /* ------------------------ IMPL ------------------------ */ #define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ @@ -385,6 +386,15 @@ static FORCE_INLINE int tDecodeCStr(SCoder* pDecoder, const char** val) { return tDecodeCStrAndLen(pDecoder, val, &len); } +static int tDecodeCStrTo(SCoder* pDecoder, char* val) { + const char* pStr; + uint64_t len; + if (tDecodeCStrAndLen(pDecoder, &pStr, &len) < 0) return -1; + + memcpy(val, pStr, len+1); + return 0; +} + #ifdef __cplusplus } #endif From 62b60991f39b92b6b86f64ca90839b6819d818b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 06:04:15 +0000 Subject: [PATCH 87/97] add encode test --- source/util/test/CMakeLists.txt | 4 + source/util/test/encodeTest.cpp | 166 ++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 source/util/test/encodeTest.cpp diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index bfc3906f79..4b6311022d 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -41,4 +41,8 @@ target_sources(freelistTest ) target_link_libraries(freelistTest os util gtest gtest_main) +# encodeTest +add_executable(encodeTest "encodeTest.cpp") +target_link_libraries(encodeTest os util gtest gtest_main) + diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp new file mode 100644 index 0000000000..e89c84d8d2 --- /dev/null +++ b/source/util/test/encodeTest.cpp @@ -0,0 +1,166 @@ +#include + +#include "gtest/gtest.h" + +#include "encode.h" + +#define BUF_SIZE 64 +td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN}; + +static int encode(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } +static int encode(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } +static int encode(SCoder *pCoder, int16_t val) { return tEncodeI16(pCoder, val); } +static int encode(SCoder *pCoder, uint16_t val) { return tEncodeU16(pCoder, val); } +static int encode(SCoder *pCoder, int32_t val) { return tEncodeI32(pCoder, val); } +static int encode(SCoder *pCoder, uint32_t val) { return tEncodeU32(pCoder, val); } +static int encode(SCoder *pCoder, int64_t val) { return tEncodeI64(pCoder, val); } +static int encode(SCoder *pCoder, uint64_t val) { return tEncodeU64(pCoder, val); } + +static int decode(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } +static int decode(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } +static int decode(SCoder *pCoder, int16_t *val) { return tDecodeI16(pCoder, val); } +static int decode(SCoder *pCoder, uint16_t *val) { return tDecodeU16(pCoder, val); } +static int decode(SCoder *pCoder, int32_t *val) { return tDecodeI32(pCoder, val); } +static int decode(SCoder *pCoder, uint32_t *val) { return tDecodeU32(pCoder, val); } +static int decode(SCoder *pCoder, int64_t *val) { return tDecodeI64(pCoder, val); } +static int decode(SCoder *pCoder, uint64_t *val) { return tDecodeU64(pCoder, val); } + +static int encodev(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } +static int encodev(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } +static int encodev(SCoder *pCoder, int16_t val) { return tEncodeI16v(pCoder, val); } +static int encodev(SCoder *pCoder, uint16_t val) { return tEncodeU16v(pCoder, val); } +static int encodev(SCoder *pCoder, int32_t val) { return tEncodeI32v(pCoder, val); } +static int encodev(SCoder *pCoder, uint32_t val) { return tEncodeU32v(pCoder, val); } +static int encodev(SCoder *pCoder, int64_t val) { return tEncodeI64v(pCoder, val); } +static int encodev(SCoder *pCoder, uint64_t val) { return tEncodeU64v(pCoder, val); } + +static int decodev(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } +static int decodev(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } +static int decodev(SCoder *pCoder, int16_t *val) { return tDecodeI16v(pCoder, val); } +static int decodev(SCoder *pCoder, uint16_t *val) { return tDecodeU16v(pCoder, val); } +static int decodev(SCoder *pCoder, int32_t *val) { return tDecodeI32v(pCoder, val); } +static int decodev(SCoder *pCoder, uint32_t *val) { return tDecodeU32v(pCoder, val); } +static int decodev(SCoder *pCoder, int64_t *val) { return tDecodeI64v(pCoder, val); } +static int decodev(SCoder *pCoder, uint64_t *val) { return tDecodeU64v(pCoder, val); } + +template +static void simple_encode_decode_func(bool var_len) { + uint8_t buf[BUF_SIZE]; + SCoder coder; + T min_val, max_val; + T step = 1; + + if (typeid(T) == typeid(int8_t)) { + min_val = INT8_MIN; + max_val = INT8_MAX; + step = 1; + } else if (typeid(T) == typeid(uint8_t)) { + min_val = 0; + max_val = UINT8_MAX; + step = 1; + } else if (typeid(T) == typeid(int16_t)) { + min_val = INT16_MIN; + max_val = INT16_MAX; + step = 1; + } else if (typeid(T) == typeid(uint16_t)) { + min_val = 0; + max_val = UINT16_MAX; + step = 1; + } else if (typeid(T) == typeid(int32_t)) { + min_val = INT32_MIN; + max_val = INT32_MAX; + step = ((T)1) << 16; + } else if (typeid(T) == typeid(uint32_t)) { + min_val = 0; + max_val = UINT32_MAX; + step = ((T)1) << 16; + } else if (typeid(T) == typeid(int64_t)) { + min_val = INT64_MIN; + max_val = INT64_MAX; + step = ((T)1) << 48; + } else if (typeid(T) == typeid(uint64_t)) { + min_val = 0; + max_val = UINT64_MAX; + step = ((T)1) << 48; + } + + T i = min_val; + for (;; /*T i = min_val; i <= max_val; i += step*/) { + T dval; + + // Encode NULL + for (td_endian_t endian : endian_arr) { + tCoderInit(&coder, endian, NULL, 0, TD_ENCODER); + + if (var_len) { + GTEST_ASSERT_EQ(encodev(&coder, i), 0); + } else { + GTEST_ASSERT_EQ(encode(&coder, i), 0); + GTEST_ASSERT_EQ(coder.pos, sizeof(T)); + } + + tCoderClear(&coder); + } + + // Encode and decode + for (td_endian_t e_endian : endian_arr) { + for (td_endian_t d_endian : endian_arr) { + // Encode + tCoderInit(&coder, e_endian, buf, BUF_SIZE, TD_ENCODER); + + if (var_len) { + GTEST_ASSERT_EQ(encodev(&coder, i), 0); + } else { + GTEST_ASSERT_EQ(encode(&coder, i), 0); + GTEST_ASSERT_EQ(coder.pos, sizeof(T)); + } + + tCoderClear(&coder); + // Decode + tCoderInit(&coder, d_endian, buf, BUF_SIZE, TD_DECODER); + + if (var_len) { + GTEST_ASSERT_EQ(decodev(&coder, &dval), 0); + } else { + GTEST_ASSERT_EQ(decode(&coder, &dval), 0); + GTEST_ASSERT_EQ(coder.pos, sizeof(T)); + } + + if (typeid(T) == typeid(int8_t) || typeid(T) == typeid(uint8_t) || e_endian == d_endian) { + GTEST_ASSERT_EQ(i, dval); + } + + tCoderClear(&coder); + } + } + + if (i == max_val) break; + + if (max_val - i < step) { + i = max_val; + } else { + i = i + step; + } + } +} + +TEST(td_encode_test, simple_encode_and_decode_test) { + uint8_t buf[10]; + SCoder encoder, decoder; + + simple_encode_decode_func(false); + simple_encode_decode_func(false); + simple_encode_decode_func(false); + simple_encode_decode_func(false); + simple_encode_decode_func(false); + simple_encode_decode_func(false); + simple_encode_decode_func(false); + simple_encode_decode_func(false); + + simple_encode_decode_func(true); + simple_encode_decode_func(true); + simple_encode_decode_func(true); + simple_encode_decode_func(true); + simple_encode_decode_func(true); + simple_encode_decode_func(true); +} \ No newline at end of file From ad3374477197c399db8d12df658a4d0417862c76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 06:15:52 +0000 Subject: [PATCH 88/97] Fix variant encoding bug --- include/util/encode.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index 02edbe888e..42548c03e3 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -141,7 +141,7 @@ static int tDecodeCStrTo(SCoder* pDecoder, char* val); return 0; #define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \ - while ((VAL) > ENCODE_LIMIT) { \ + while ((VAL) >= ENCODE_LIMIT) { \ if ((CODER)->data) { \ if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \ @@ -391,7 +391,7 @@ static int tDecodeCStrTo(SCoder* pDecoder, char* val) { uint64_t len; if (tDecodeCStrAndLen(pDecoder, &pStr, &len) < 0) return -1; - memcpy(val, pStr, len+1); + memcpy(val, pStr, len + 1); return 0; } From 99510b689c3e7f1c8e7bda3bd8d75ab745aeabf9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 06:18:07 +0000 Subject: [PATCH 89/97] add more check --- source/util/test/encodeTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index e89c84d8d2..0e276e4338 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -115,6 +115,8 @@ static void simple_encode_decode_func(bool var_len) { GTEST_ASSERT_EQ(coder.pos, sizeof(T)); } + int32_t epos = coder.pos; + tCoderClear(&coder); // Decode tCoderInit(&coder, d_endian, buf, BUF_SIZE, TD_DECODER); @@ -126,6 +128,8 @@ static void simple_encode_decode_func(bool var_len) { GTEST_ASSERT_EQ(coder.pos, sizeof(T)); } + GTEST_ASSERT_EQ(coder.pos, epos); + if (typeid(T) == typeid(int8_t) || typeid(T) == typeid(uint8_t) || e_endian == d_endian) { GTEST_ASSERT_EQ(i, dval); } From 8aaf22ae7549b1f306d6fb53f63f877174896c42 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 06:21:33 +0000 Subject: [PATCH 90/97] more --- source/util/test/encodeTest.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 0e276e4338..59a37386d3 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -148,10 +148,7 @@ static void simple_encode_decode_func(bool var_len) { } } -TEST(td_encode_test, simple_encode_and_decode_test) { - uint8_t buf[10]; - SCoder encoder, decoder; - +TEST(td_encode_test, encode_decode_fixed_len_integer) { simple_encode_decode_func(false); simple_encode_decode_func(false); simple_encode_decode_func(false); @@ -160,7 +157,9 @@ TEST(td_encode_test, simple_encode_and_decode_test) { simple_encode_decode_func(false); simple_encode_decode_func(false); simple_encode_decode_func(false); +} +TEST(td_encode_test, encode_decode_variant_len_integer) { simple_encode_decode_func(true); simple_encode_decode_func(true); simple_encode_decode_func(true); From 7b3a04c4dd196e896ab66ccbf70a72e736e87e98 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 06:34:54 +0000 Subject: [PATCH 91/97] more --- include/util/encode.h | 8 ++++---- source/util/test/encodeTest.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index 42548c03e3..31d386c450 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -100,7 +100,7 @@ static int tEncodeI64v(SCoder* pEncoder, int64_t val); static int tEncodeFloat(SCoder* pEncoder, float val); static int tEncodeDouble(SCoder* pEncoder, double val); static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); -static int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len); +static int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); static int tEncodeCStr(SCoder* pEncoder, const char* val); /* ------------------------ DECODE ------------------------ */ @@ -141,7 +141,7 @@ static int tDecodeCStrTo(SCoder* pDecoder, char* val); return 0; #define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \ - while ((VAL) >= ENCODE_LIMIT) { \ + while ((VAL) >= ENCODE_LIMIT) { \ if ((CODER)->data) { \ if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \ TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \ @@ -261,12 +261,12 @@ static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_ return 0; } -static FORCE_INLINE int tEncodeCstrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { +static FORCE_INLINE int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { return tEncodeBinary(pEncoder, (void*)val, len + 1); } static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { - return tEncodeCstrWithLen(pEncoder, val, (uint64_t)strlen(val)); + return tEncodeCStrWithLen(pEncoder, val, (uint64_t)strlen(val)); } /* ------------------------ FOR DECODER ------------------------ */ diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 59a37386d3..dc10cec443 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -166,4 +166,36 @@ TEST(td_encode_test, encode_decode_variant_len_integer) { simple_encode_decode_func(true); simple_encode_decode_func(true); simple_encode_decode_func(true); +} + +TEST(td_encode_test, encode_decode_cstr) { + uint8_t * buf = new uint8_t[1024 * 1024]; + char * cstr = new char[1024 * 1024]; + const char *dcstr; + SCoder encoder; + SCoder decoder; + + for (size_t i = 0; i < 1024 * 2 - 1; i++) { + memset(cstr, 'a', i); + cstr[i] = '\0'; + for (td_endian_t endian : endian_arr) { + // Encode + tCoderInit(&encoder, endian, buf, 1024 * 1024, TD_ENCODER); + + GTEST_ASSERT_EQ(tEncodeCStr(&encoder, cstr), 0); + + tCoderClear(&encoder); + + // Decode + tCoderInit(&decoder, endian, buf, 1024 * 1024, TD_DECODER); + + GTEST_ASSERT_EQ(tDecodeCStr(&decoder, &dcstr), 0); + GTEST_ASSERT_EQ(memcmp(dcstr, cstr, i + 1), 0); + + tCoderClear(&decoder); + } + } + + delete buf; + delete cstr; } \ No newline at end of file From 086a2202ce9c807f776bf4cc8e2f9385760ceba2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 08:45:31 +0000 Subject: [PATCH 92/97] more --- include/util/encode.h | 49 ++++---- source/util/src/encode.c | 21 ++-- source/util/test/encodeTest.cpp | 199 ++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 32 deletions(-) diff --git a/include/util/encode.h b/include/util/encode.h index 31d386c450..ba63759737 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -81,31 +81,32 @@ void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, void tCoderClear(SCoder* pCoder); /* ------------------------ ENCODE ------------------------ */ -static int tStartEncode(SCoder* pEncoder); -static void tEndEncode(SCoder* pEncoder); -static int tEncodeU8(SCoder* pEncoder, uint8_t val); -static int tEncodeI8(SCoder* pEncoder, int8_t val); -static int tEncodeU16(SCoder* pEncoder, uint16_t val); -static int tEncodeI16(SCoder* pEncoder, int16_t val); -static int tEncodeU32(SCoder* pEncoder, uint32_t val); -static int tEncodeI32(SCoder* pEncoder, int32_t val); -static int tEncodeU64(SCoder* pEncoder, uint64_t val); -static int tEncodeI64(SCoder* pEncoder, int64_t val); -static int tEncodeU16v(SCoder* pEncoder, uint16_t val); -static int tEncodeI16v(SCoder* pEncoder, int16_t val); -static int tEncodeU32v(SCoder* pEncoder, uint32_t val); -static int tEncodeI32v(SCoder* pEncoder, int32_t val); -static int tEncodeU64v(SCoder* pEncoder, uint64_t val); -static int tEncodeI64v(SCoder* pEncoder, int64_t val); -static int tEncodeFloat(SCoder* pEncoder, float val); -static int tEncodeDouble(SCoder* pEncoder, double val); -static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); -static int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); -static int tEncodeCStr(SCoder* pEncoder, const char* val); +int tStartEncode(SCoder* pEncoder); +void tEndEncode(SCoder* pEncoder); +static int tEncodeU8(SCoder* pEncoder, uint8_t val); +static int tEncodeI8(SCoder* pEncoder, int8_t val); +static int tEncodeU16(SCoder* pEncoder, uint16_t val); +static int tEncodeI16(SCoder* pEncoder, int16_t val); +static int tEncodeU32(SCoder* pEncoder, uint32_t val); +static int tEncodeI32(SCoder* pEncoder, int32_t val); +static int tEncodeU64(SCoder* pEncoder, uint64_t val); +static int tEncodeI64(SCoder* pEncoder, int64_t val); +static int tEncodeU16v(SCoder* pEncoder, uint16_t val); +static int tEncodeI16v(SCoder* pEncoder, int16_t val); +static int tEncodeU32v(SCoder* pEncoder, uint32_t val); +static int tEncodeI32v(SCoder* pEncoder, int32_t val); +static int tEncodeU64v(SCoder* pEncoder, uint64_t val); +static int tEncodeI64v(SCoder* pEncoder, int64_t val); +static int tEncodeFloat(SCoder* pEncoder, float val); +static int tEncodeDouble(SCoder* pEncoder, double val); +static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); +static int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); +static int tEncodeCStr(SCoder* pEncoder, const char* val); /* ------------------------ DECODE ------------------------ */ -static int tStartDecode(SCoder* pDecoder); -static void tEndDecode(SCoder* pDecoder); +int tStartDecode(SCoder* pDecoder); +void tEndDecode(SCoder* pDecoder); +static bool tDecodeIsEnd(SCoder* pCoder); static int tDecodeU8(SCoder* pDecoder, uint8_t* val); static int tDecodeI8(SCoder* pDecoder, int8_t* val); static int tDecodeU16(SCoder* pDecoder, uint16_t* val); @@ -395,6 +396,8 @@ static int tDecodeCStrTo(SCoder* pDecoder, char* val) { return 0; } +static FORCE_INLINE bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } + #ifdef __cplusplus } #endif diff --git a/source/util/src/encode.c b/source/util/src/encode.c index a3946daf5c..24d033f697 100644 --- a/source/util/src/encode.c +++ b/source/util/src/encode.c @@ -47,8 +47,6 @@ void tCoderClear(SCoder* pCoder) { } } -bool tDecodeIsEnd(SCoder* pCoder) { return (pCoder->size == pCoder->pos); } - int tStartEncode(SCoder* pCoder) { struct SCoderNode* pNode; @@ -62,7 +60,7 @@ int tStartEncode(SCoder* pCoder) { pNode->pos = pCoder->pos; pNode->size = pCoder->size; - pCoder->data = pNode->data + sizeof(int32_t); + pCoder->data = pNode->data + pNode->pos + sizeof(int32_t); pCoder->pos = 0; pCoder->size = pNode->size - pNode->pos - sizeof(int32_t); @@ -75,27 +73,32 @@ int tStartEncode(SCoder* pCoder) { void tEndEncode(SCoder* pCoder) { struct SCoderNode* pNode; + int32_t len; if (pCoder->data) { pNode = TD_SLIST_HEAD(&(pCoder->stack)); ASSERT(pNode); TD_SLIST_POP(&(pCoder->stack)); - // TODO: tEncodeI32(pNode, pCoder->pos); + len = pCoder->pos; pCoder->data = pNode->data; pCoder->size = pNode->size; - pCoder->pos = pNode->pos + pCoder->pos; + pCoder->pos = pNode->pos; + + tEncodeI32(pCoder, len); + + TD_CODER_MOVE_POS(pCoder, len); free(pNode); } } int tStartDecode(SCoder* pCoder) { - int32_t size; + int32_t len; struct SCoderNode* pNode; - // TODO: if (tDecodeI32(pCoder, &size) < 0) return -1; + if (tDecodeI32(pCoder, &len) < 0) return -1; pNode = malloc(sizeof(*pNode)); if (pNode == NULL) return -1; @@ -104,8 +107,8 @@ int tStartDecode(SCoder* pCoder) { pNode->pos = pCoder->pos; pNode->size = pCoder->size; - pCoder->data = pCoder->data; - pCoder->size = size; + pCoder->data = pNode->data + pNode->pos; + pCoder->size = len; pCoder->pos = 0; TD_SLIST_PUSH(&(pCoder->stack), pNode); diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index dc10cec443..b0e970eb07 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -198,4 +198,203 @@ TEST(td_encode_test, encode_decode_cstr) { delete buf; delete cstr; +} + +typedef struct { + int32_t A_a; + int64_t A_b; + char * A_c; +} SStructA_v1; + +static int tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32(pCoder, pSAV1->A_a) < 0) return -1; + if (tEncodeI64(pCoder, pSAV1->A_b) < 0) return -1; + if (tEncodeCStr(pCoder, pSAV1->A_c) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32(pCoder, &pSAV1->A_a) < 0) return -1; + if (tDecodeI64(pCoder, &pSAV1->A_b) < 0) return -1; + const char *tstr; + uint64_t len; + if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; + pSAV1->A_c = (char *)TCODER_MALLOC(len + 1, pCoder); + memcpy(pSAV1->A_c, tstr, len + 1); + + tEndDecode(pCoder); + return 0; +} + +typedef struct { + int32_t A_a; + int64_t A_b; + char * A_c; + // -------------------BELOW FEILDS ARE ADDED IN A NEW VERSION-------------- + int16_t A_d; + int16_t A_e; +} SStructA_v2; + +static int tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32(pCoder, pSAV2->A_a) < 0) return -1; + if (tEncodeI64(pCoder, pSAV2->A_b) < 0) return -1; + if (tEncodeCStr(pCoder, pSAV2->A_c) < 0) return -1; + + // ------------------------NEW FIELDS ENCODE------------------------------- + if (tEncodeI16(pCoder, pSAV2->A_d) < 0) return -1; + if (tEncodeI16(pCoder, pSAV2->A_e) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32(pCoder, &pSAV2->A_a) < 0) return -1; + if (tDecodeI64(pCoder, &pSAV2->A_b) < 0) return -1; + const char *tstr; + uint64_t len; + if (tDecodeCStrAndLen(pCoder, &tstr, &len) < 0) return -1; + pSAV2->A_c = (char *)TCODER_MALLOC(len + 1, pCoder); + memcpy(pSAV2->A_c, tstr, len + 1); + + // ------------------------NEW FIELDS DECODE------------------------------- + if (!tDecodeIsEnd(pCoder)) { + if (tDecodeI16(pCoder, &pSAV2->A_d) < 0) return -1; + if (tDecodeI16(pCoder, &pSAV2->A_e) < 0) return -1; + } else { + pSAV2->A_d = 0; + pSAV2->A_e = 0; + } + + tEndDecode(pCoder); + return 0; +} + +typedef struct { + SStructA_v1 *pA; + int32_t v_a; + int8_t v_b; +} SFinalReq_v1; + +static int tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tSStructA_v1_encode(pCoder, ps1->pA) < 0) return -1; + if (tEncodeI32(pCoder, ps1->v_a) < 0) return -1; + if (tEncodeI8(pCoder, ps1->v_b) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { + if (tStartDecode(pCoder) < 0) return -1; + + ps1->pA = (SStructA_v1 *)TCODER_MALLOC(sizeof(*(ps1->pA)), pCoder); + if (tSStructA_v1_decode(pCoder, ps1->pA) < 0) return -1; + if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1; + if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +typedef struct { + SStructA_v2 *pA; + int32_t v_a; + int8_t v_b; + // ----------------------- Feilds added ----------------------- + int16_t v_c; +} SFinalReq_v2; + +static int tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tSStructA_v2_encode(pCoder, ps2->pA) < 0) return -1; + if (tEncodeI32(pCoder, ps2->v_a) < 0) return -1; + if (tEncodeI8(pCoder, ps2->v_b) < 0) return -1; + + // ----------------------- Feilds added encode ----------------------- + if (tEncodeI16(pCoder, ps2->v_c) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +static int tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { + if (tStartDecode(pCoder) < 0) return -1; + + ps2->pA = (SStructA_v2 *)TCODER_MALLOC(sizeof(*(ps2->pA)), pCoder); + if (tSStructA_v2_decode(pCoder, ps2->pA) < 0) return -1; + if (tDecodeI32(pCoder, &ps2->v_a) < 0) return -1; + if (tDecodeI8(pCoder, &ps2->v_b) < 0) return -1; + + // ----------------------- Feilds added decode ----------------------- + if (tDecodeIsEnd(pCoder)) { + ps2->v_c = 0; + } else { + if (tDecodeI16(pCoder, &ps2->v_c) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +TEST(td_encode_test, compound_struct_encode_test) { + SCoder encoder, decoder; + uint8_t * buf1; + int32_t buf1size; + uint8_t * buf2; + int32_t buf2size; + SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = "Hello"}; + SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = "Hello", .A_d = 67, .A_e = 13}; + SFinalReq_v1 req1 = {.pA = &sa1, .v_a = 15, .v_b = 35}; + SFinalReq_v2 req2 = {.pA = &sa2, .v_a = 15, .v_b = 32, .v_c = 37}; + SFinalReq_v1 dreq1; + SFinalReq_v2 dreq21, dreq22; + + // Get size + tCoderInit(&encoder, TD_LITTLE_ENDIAN, nullptr, 0, TD_ENCODER); + GTEST_ASSERT_EQ(tSFinalReq_v1_encode(&encoder, &req1), 0); + buf1size = encoder.pos; + buf1 = new uint8_t[encoder.pos]; + tCoderClear(&encoder); + + tCoderInit(&encoder, TD_LITTLE_ENDIAN, nullptr, 0, TD_ENCODER); + GTEST_ASSERT_EQ(tSFinalReq_v2_encode(&encoder, &req2), 0); + buf2size = encoder.pos; + buf2 = new uint8_t[encoder.pos]; + tCoderClear(&encoder); + + // Encode + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_ENCODER); + GTEST_ASSERT_EQ(tSFinalReq_v1_encode(&encoder, &req1), 0); + tCoderClear(&encoder); + + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf2, buf2size, TD_ENCODER); + GTEST_ASSERT_EQ(tSFinalReq_v2_encode(&encoder, &req2), 0); + tCoderClear(&encoder); + + // Decode + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_DECODER); + GTEST_ASSERT_EQ(tSFinalReq_v1_decode(&decoder, &dreq1), 0); + tCoderClear(&decoder); + + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_DECODER); + GTEST_ASSERT_EQ(tSFinalReq_v2_decode(&decoder, &dreq21), 0); + tCoderClear(&decoder); + + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf2, buf2size, TD_DECODER); + GTEST_ASSERT_EQ(tSFinalReq_v2_decode(&decoder, &dreq22), 0); + tCoderClear(&decoder); } \ No newline at end of file From 97d26ef10d28cc41620e0819d2902dc0d484a9f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 09:34:20 +0000 Subject: [PATCH 93/97] fix bug --- source/util/src/encode.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/util/src/encode.c b/source/util/src/encode.c index 24d033f697..71985845af 100644 --- a/source/util/src/encode.c +++ b/source/util/src/encode.c @@ -86,7 +86,11 @@ void tEndEncode(SCoder* pCoder) { pCoder->size = pNode->size; pCoder->pos = pNode->pos; - tEncodeI32(pCoder, len); + if (TD_RT_ENDIAN() == pCoder->endian) { + tPut(int32_t, pCoder->data + pCoder->pos, len); + } else { + tRPut32(pCoder->data + pCoder->pos, len); + } TD_CODER_MOVE_POS(pCoder, len); From 1fde98815b40f4819ae89f2a3a8d98763e840a3e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 10:02:21 +0000 Subject: [PATCH 94/97] refact and fix some bugs --- source/util/src/encode.c | 10 +++++++--- source/util/test/encodeTest.cpp | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/util/src/encode.c b/source/util/src/encode.c index 71985845af..40c03ea051 100644 --- a/source/util/src/encode.c +++ b/source/util/src/encode.c @@ -50,6 +50,7 @@ void tCoderClear(SCoder* pCoder) { int tStartEncode(SCoder* pCoder) { struct SCoderNode* pNode; + ASSERT(pCoder->type == TD_ENCODER); if (pCoder->data) { if (pCoder->size - pCoder->pos < sizeof(int32_t)) return -1; @@ -75,6 +76,7 @@ void tEndEncode(SCoder* pCoder) { struct SCoderNode* pNode; int32_t len; + ASSERT(pCoder->type == TD_ENCODER); if (pCoder->data) { pNode = TD_SLIST_HEAD(&(pCoder->stack)); ASSERT(pNode); @@ -92,7 +94,7 @@ void tEndEncode(SCoder* pCoder) { tRPut32(pCoder->data + pCoder->pos, len); } - TD_CODER_MOVE_POS(pCoder, len); + TD_CODER_MOVE_POS(pCoder, len + sizeof(int32_t)); free(pNode); } @@ -102,6 +104,7 @@ int tStartDecode(SCoder* pCoder) { int32_t len; struct SCoderNode* pNode; + ASSERT(pCoder->type == TD_DECODER); if (tDecodeI32(pCoder, &len) < 0) return -1; pNode = malloc(sizeof(*pNode)); @@ -121,10 +124,11 @@ int tStartDecode(SCoder* pCoder) { } void tEndDecode(SCoder* pCoder) { - ASSERT(tDecodeIsEnd(pCoder)); - struct SCoderNode* pNode; + ASSERT(pCoder->type == TD_DECODER); + ASSERT(tDecodeIsEnd(pCoder)); + pNode = TD_SLIST_HEAD(&(pCoder->stack)); ASSERT(pNode); TD_SLIST_POP(&(pCoder->stack)); diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index b0e970eb07..ec4d9cd182 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -305,7 +305,7 @@ static int tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tDecodeI32(pCoder, &ps1->v_a) < 0) return -1; if (tDecodeI8(pCoder, &ps1->v_b) < 0) return -1; - tEndEncode(pCoder); + tEndDecode(pCoder); return 0; } @@ -346,7 +346,7 @@ static int tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tDecodeI16(pCoder, &ps2->v_c) < 0) return -1; } - tEndEncode(pCoder); + tEndDecode(pCoder); return 0; } From 395b2bbeab93586d1616778544e507cfa3810b61 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 3 Jan 2022 10:07:42 +0000 Subject: [PATCH 95/97] add more test --- source/util/test/encodeTest.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index ec4d9cd182..1a861701d7 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -388,13 +388,34 @@ TEST(td_encode_test, compound_struct_encode_test) { // Decode tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_DECODER); GTEST_ASSERT_EQ(tSFinalReq_v1_decode(&decoder, &dreq1), 0); + GTEST_ASSERT_EQ(dreq1.pA->A_a, req1.pA->A_a); + GTEST_ASSERT_EQ(dreq1.pA->A_b, req1.pA->A_b); + GTEST_ASSERT_EQ(strcmp(dreq1.pA->A_c, req1.pA->A_c), 0); + GTEST_ASSERT_EQ(dreq1.v_a, req1.v_a); + GTEST_ASSERT_EQ(dreq1.v_b, req1.v_b); tCoderClear(&decoder); tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf1, buf1size, TD_DECODER); GTEST_ASSERT_EQ(tSFinalReq_v2_decode(&decoder, &dreq21), 0); + GTEST_ASSERT_EQ(dreq21.pA->A_a, req1.pA->A_a); + GTEST_ASSERT_EQ(dreq21.pA->A_b, req1.pA->A_b); + GTEST_ASSERT_EQ(strcmp(dreq21.pA->A_c, req1.pA->A_c), 0); + GTEST_ASSERT_EQ(dreq21.pA->A_d, 0); + GTEST_ASSERT_EQ(dreq21.pA->A_e, 0); + GTEST_ASSERT_EQ(dreq21.v_a, req1.v_a); + GTEST_ASSERT_EQ(dreq21.v_b, req1.v_b); + GTEST_ASSERT_EQ(dreq21.v_c, 0); tCoderClear(&decoder); tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf2, buf2size, TD_DECODER); GTEST_ASSERT_EQ(tSFinalReq_v2_decode(&decoder, &dreq22), 0); + GTEST_ASSERT_EQ(dreq22.pA->A_a, req2.pA->A_a); + GTEST_ASSERT_EQ(dreq22.pA->A_b, req2.pA->A_b); + GTEST_ASSERT_EQ(strcmp(dreq22.pA->A_c, req2.pA->A_c), 0); + GTEST_ASSERT_EQ(dreq22.pA->A_d, req2.pA->A_d); + GTEST_ASSERT_EQ(dreq22.pA->A_e, req2.pA->A_e); + GTEST_ASSERT_EQ(dreq22.v_a, req2.v_a); + GTEST_ASSERT_EQ(dreq22.v_b, req2.v_b); + GTEST_ASSERT_EQ(dreq22.v_c, req2.v_c); tCoderClear(&decoder); } \ No newline at end of file From 88ba5faec9ff06f6f492cec61b6301e164ce9dd2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 3 Jan 2022 22:12:28 +0800 Subject: [PATCH 96/97] refactor code --- .../index/inc/index_fst_counting_writer.h | 1 + source/libs/index/src/index.c | 8 +- source/libs/index/src/index_cache.c | 2 +- .../index/src/index_fst_counting_writer.c | 7 ++ source/libs/index/src/index_tfile.c | 90 +++++++++++-------- source/libs/index/test/indexTests.cc | 35 +++++--- 6 files changed, 89 insertions(+), 54 deletions(-) diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 1504a69a08..fcc0d5a0b3 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -38,6 +38,7 @@ typedef struct WriterCtx { int fd; bool readOnly; char buf[256]; + int size; } file; struct { int32_t capa; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9c7320b301..ff973dd7e3 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -135,7 +135,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { SIndexTerm* p = taosArrayGetP(fVals, i); char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName}; + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); @@ -150,7 +150,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { SIndexTerm* p = taosArrayGetP(fVals, i); char buf[128] = {0}; - ICacheKey key = {.suid = p->suid, .colName = p->colName}; + ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName)}; int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** cache = taosHashGet(index->colObj, buf, sz); @@ -212,7 +212,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result indexInterResultsDestroy(interResults); #endif - return 1; + return 0; } int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { @@ -310,7 +310,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result pthread_mutex_lock(&sIdx->mtx); char buf[128] = {0}; - ICacheKey key = {.suid = term->suid, .colName = term->colName}; + ICacheKey key = {.suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName)}; int32_t sz = indexSerialCacheKey(&key, buf); IndexCache** pCache = taosHashGet(sIdx->colObj, buf, sz); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index b4c533e998..e95de9286e 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 10 * 10000 +#define MEM_TERM_LIMIT 5 * 10000 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 7906dfea11..fb730d5f70 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -72,9 +72,15 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int if (readOnly == false) { // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenCreateWriteAppend(path); + struct stat fstat; + stat(path, &fstat); + ctx->file.size = fstat.st_size; } else { // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenRead(path); + struct stat fstat; + stat(path, &fstat); + ctx->file.size = fstat.st_size; } memcpy(ctx->file.buf, path, strlen(path)); if (ctx->file.fd < 0) { @@ -104,6 +110,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { tfClose(ctx->file.fd); + ctx->flush(ctx); if (remove) { unlink(ctx->file.buf); } } free(ctx); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 95c713fb0a..9c4d1d1139 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2019 TAOS Data, Inc. - * +p * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. @@ -45,12 +45,13 @@ static int tfileReaderLoadHeader(TFileReader* reader); static int tfileReaderLoadFst(TFileReader* reader); static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); -static int tfileGetFileList(const char* path, SArray* result); -static int tfileRmExpireFile(SArray* result); -static void tfileDestroyFileName(void* elem); -static int tfileCompare(const void* a, const void* b); -static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); -static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); +static SArray* tfileGetFileList(const char* path); +static int tfileRmExpireFile(SArray* result); +static void tfileDestroyFileName(void* elem); +static int tfileCompare(const void* a, const void* b); +static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version); +static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version); +static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version); TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); @@ -59,21 +60,22 @@ TFileCache* tfileCacheCreate(const char* path) { tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; - SArray* files = taosArrayInit(4, sizeof(void*)); - tfileGetFileList(path, files); - taosArraySort(files, tfileCompare); - tfileRmExpireFile(files); + SArray* files = tfileGetFileList(path); uint64_t suid; int32_t colId, version; for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); - if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) { + char colName[256] = {0}; + if (0 != tfileParseFileName(file, &suid, colName, (int*)&version)) { indexInfo("try parse invalid file: %s, skip it", file); continue; } + // use version info later + char fullName[256] = {0}; + sprintf(fullName, "%s/%s", path, file); - WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 1024 * 64); + WriterCtx* wc = writerCtxCreate(TFile, fullName, true, 1024 * 1024 * 64); if (wc == NULL) { indexError("failed to open index:%s", file); goto End; @@ -200,12 +202,9 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul } TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { - char filename[128] = {0}; - int32_t coldId = 1; - tfileGenFileName(filename, suid, coldId, version); - char fullname[256] = {0}; - snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); + tfileGenFileFullName(fullname, path, suid, colName, version); + indexInfo("open write file name %s", fullname); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); if (wcx == NULL) { return NULL; } @@ -218,13 +217,11 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c return tfileWriterCreate(wcx, &tfh); } TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName) { - char filename[128] = {0}; - int32_t coldId = 1; - tfileGenFileName(filename, suid, coldId, version); - char fullname[256] = {0}; - snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); + tfileGenFileFullName(fullname, path, suid, colName, version); + WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); + indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); if (wc == NULL) { return NULL; } TFileReader* reader = tfileReaderCreate(wc); @@ -324,7 +321,6 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { } // write data - indexError("--------Begin----------------"); for (size_t i = 0; i < sz; i++) { // TODO, fst batch write later TFileValue* v = taosArrayGetP((SArray*)data, i); @@ -332,11 +328,10 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); } else { - indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, - (int)taosArrayGetSize(v->tableId)); + // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset, + // (int)taosArrayGetSize(v->tableId)); } } - indexError("--------End----------------"); fstBuilderFinish(tw->fb); fstBuilderDestroy(tw->fb); tw->fb = NULL; @@ -550,6 +545,9 @@ static int tfileReaderLoadHeader(TFileReader* reader) { // indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), errno, reader->ctx->file.fd, reader->ctx->file.buf); + } else { + indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.fd, reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); @@ -558,13 +556,14 @@ static int tfileReaderLoadHeader(TFileReader* reader) { } static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later - static int FST_MAX_SIZE = 64 * 1024; + static int FST_MAX_SIZE = 64 * 1024 * 1024; char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); + indexError("nread = %d, and fst offset=%d, filename: %s ", nread, reader->header.fstOffset, ctx->file.buf); // we assuse fst size less than FST_MAX_SIZE assert(nread > 0 && nread < FST_MAX_SIZE); @@ -608,19 +607,26 @@ void tfileReaderUnRef(TFileReader* reader) { } } -static int tfileGetFileList(const char* path, SArray* result) { +static SArray* tfileGetFileList(const char* path) { + SArray* files = taosArrayInit(4, sizeof(void*)); + DIR* dir = opendir(path); - if (NULL == dir) { return -1; } + if (NULL == dir) { return NULL; } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { + if (entry->d_type && DT_DIR) { continue; } size_t len = strlen(entry->d_name); char* buf = calloc(1, len + 1); memcpy(buf, entry->d_name, len); - taosArrayPush(result, &buf); + taosArrayPush(files, &buf); } closedir(dir); - return 0; + + taosArraySort(files, tfileCompare); + tfileRmExpireFile(files); + + return files; } static int tfileRmExpireFile(SArray* result) { // TODO(yihao): remove expire tindex after restart @@ -641,15 +647,21 @@ static int tfileCompare(const void* a, const void* b) { if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } -// tfile name suid-colId-version.tindex -static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version) { - sprintf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version); - return; -} -static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) { - if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { + +static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version) { + if (3 == sscanf(filename, "%" PRIu64 "-%[^-]-%d.tindex", suid, col, version)) { // read suid & colid & version success return 0; } return -1; } +// tfile name suid-colId-version.tindex +static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int version) { + sprintf(filename, "%" PRIu64 "-%s-%d.tindex", suid, col, version); + return; +} +static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version) { + char filename[128] = {0}; + tfileGenFileName(filename, suid, col, version); + sprintf(fullname, "%s/%s", path, filename); +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index bdfb86ce17..3a8a880b3b 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -13,6 +13,7 @@ * along with this program. If not, see . */ #include +#include #include #include #include @@ -638,7 +639,7 @@ class IndexObj { indexInit(); } int Init(const std::string& dir) { - taosRemoveDir(dir.c_str()); + // taosRemoveDir(dir.c_str()); taosMkDir(dir.c_str()); int ret = indexOpen(&opts, dir.c_str(), &idx); if (ret != 0) { @@ -663,10 +664,11 @@ class IndexObj { int WriteMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { std::string tColVal = colVal; + size_t colValSize = tColVal.size(); for (int i = 0; i < numOfTable; i++) { - tColVal[tColVal.size() - 1] = 'a' + i % 26; + tColVal[i % colValSize] = 'a' + i % 26; SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < 10; i++) { @@ -695,7 +697,13 @@ class IndexObj { indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); - if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; } + + int64_t s = taosGetTimestampUs(); + if (Search(mq, result) == 0) { + int64_t e = taosGetTimestampUs(); + std::cout << "search one successfully and time cost:" << e - s << std::endl; + } else { + } int sz = taosArrayGetSize(result); indexMultiTermQueryDestroy(mq); taosArrayDestroy(result); @@ -810,12 +818,12 @@ TEST_F(IndexEnv2, testIndexOpen) { } TEST_F(IndexEnv2, testIndex_TrigeFlush) { - std::string path = "/tmp/test1"; + std::string path = "/tmp/testxxx"; if (index->Init(path) != 0) { // r std::cout << "failed to init" << std::endl; } - int numOfTable = 100 * 10000; + int numOfTable = 2 * 10000; index->WriteMillonData("tag1", "Hello Wolrd", numOfTable); int target = index->SearchOne("tag1", "Hello Wolrd"); std::cout << "Get Index: " << target << std::endl; @@ -826,6 +834,10 @@ static void write_and_search(IndexObj* idx) { std::string colName("tag1"), colVal("Hello"); int target = idx->SearchOne("tag1", "Hello"); + std::cout << "search: " << target << std::endl; + target = idx->SearchOne("tag2", "Test"); + std::cout << "search: " << target << std::endl; + idx->PutOne(colName, colVal); } TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { @@ -833,7 +845,10 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { if (index->Init(path) != 0) { // opt } - index->WriteMultiMillonData("tag1", "Hello", 200000); + index->PutOne("tag1", "Hello"); + index->PutOne("tag2", "Test"); + index->WriteMultiMillonData("tag1", "Hello", 5 * 10000); + index->WriteMultiMillonData("tag2", "Test", 5 * 10000); std::thread threads[NUM_OF_THREAD]; for (int i = 0; i < NUM_OF_THREAD; i++) { @@ -847,15 +862,15 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } TEST_F(IndexEnv2, testIndex_restart) { - std::string path = "/tmp"; + std::string path = "/tmp/test1"; if (index->Init(path) != 0) {} } TEST_F(IndexEnv2, testIndex_performance) { - std::string path = "/tmp"; + std::string path = "/tmp/test2"; if (index->Init(path) != 0) {} } TEST_F(IndexEnv2, testIndexMultiTag) { - std::string path = "/tmp"; + std::string path = "/tmp/test3"; if (index->Init(path) != 0) {} } From c1a2366d673702e5939604b40e56df159f850efd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 3 Jan 2022 22:28:01 +0800 Subject: [PATCH 97/97] refactor code --- source/libs/index/src/index.c | 3 +++ source/libs/index/src/index_fst_counting_writer.c | 2 ++ source/libs/index/src/index_tfile.c | 11 +++++++---- source/libs/index/test/indexTests.cc | 6 +++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ff973dd7e3..0657c68458 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -73,6 +73,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { #ifdef USE_INVERTED_INDEX // sIdx->cache = (void*)indexCacheCreate(sIdx); sIdx->tindex = indexTFileCreate(path); + if (sIdx->tindex == NULL) { goto END; } sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->cVersion = 1; sIdx->path = calloc(1, strlen(path) + 1); @@ -83,6 +84,8 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { return 0; #endif +END: + if (sIdx != NULL) { indexClose(sIdx); } *index = NULL; return -1; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index fb730d5f70..0f29da1c27 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -72,12 +72,14 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int if (readOnly == false) { // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenCreateWriteAppend(path); + struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; } else { // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenRead(path); + struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 9c4d1d1139..d4d13ddf19 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -66,12 +66,14 @@ TFileCache* tfileCacheCreate(const char* path) { int32_t colId, version; for (size_t i = 0; i < taosArrayGetSize(files); i++) { char* file = taosArrayGetP(files, i); - char colName[256] = {0}; + + // refactor later, use colname and version info + char colName[256] = {0}; if (0 != tfileParseFileName(file, &suid, colName, (int*)&version)) { indexInfo("try parse invalid file: %s, skip it", file); continue; } - // use version info later + char fullName[256] = {0}; sprintf(fullName, "%s/%s", path, file); @@ -204,7 +206,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) { char fullname[256] = {0}; tfileGenFileFullName(fullname, path, suid, colName, version); - indexInfo("open write file name %s", fullname); + // indexInfo("open write file name %s", fullname); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); if (wcx == NULL) { return NULL; } @@ -221,7 +223,7 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c tfileGenFileFullName(fullname, path, suid, colName, version); WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); - indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); + // indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); if (wc == NULL) { return NULL; } TFileReader* reader = tfileReaderCreate(wc); @@ -356,6 +358,7 @@ IndexTFile* indexTFileCreate(const char* path) { return tfile; } void indexTFileDestroy(IndexTFile* tfile) { + if (tfile == NULL) { return; } tfileCacheDestroy(tfile->cache); free(tfile); } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 3a8a880b3b..77b6e02f18 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -823,7 +823,7 @@ TEST_F(IndexEnv2, testIndex_TrigeFlush) { // r std::cout << "failed to init" << std::endl; } - int numOfTable = 2 * 10000; + int numOfTable = 100 * 10000; index->WriteMillonData("tag1", "Hello Wolrd", numOfTable); int target = index->SearchOne("tag1", "Hello Wolrd"); std::cout << "Get Index: " << target << std::endl; @@ -847,8 +847,8 @@ TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { } index->PutOne("tag1", "Hello"); index->PutOne("tag2", "Test"); - index->WriteMultiMillonData("tag1", "Hello", 5 * 10000); - index->WriteMultiMillonData("tag2", "Test", 5 * 10000); + index->WriteMultiMillonData("tag1", "Hello", 50 * 10000); + index->WriteMultiMillonData("tag2", "Test", 50 * 10000); std::thread threads[NUM_OF_THREAD]; for (int i = 0; i < NUM_OF_THREAD; i++) {