diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8132eab539..fd0ada95d9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,7 +18,8 @@ "ms-vscode.cpptools", "ms-vscode.cmake-tools", "austin.code-gnu-global", - "visualstudioexptteam.vscodeintel" + "visualstudioexptteam.vscodeintel", + "eamodio.gitlens" ], // Use 'forwardPorts' to make a list of ports inside the container available locally. diff --git a/cmake/cmake.options b/cmake/cmake.options index 44fa8c7e4b..faa45256fb 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -56,6 +56,12 @@ option( OFF ) +option( + BUILD_WITH_TRAFT + "If build with traft" + OFF +) + option( BUILD_DEPENDENCY_TESTS "If build dependency tests" diff --git a/cmake/traft_CMakeLists.txt.in b/cmake/traft_CMakeLists.txt.in new file mode 100644 index 0000000000..9b571b3666 --- /dev/null +++ b/cmake/traft_CMakeLists.txt.in @@ -0,0 +1,14 @@ + +# traft +ExternalProject_Add(traft + GIT_REPOSITORY https://github.com/taosdata/traft.git + GIT_TAG for_3.0 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/traft" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft" + #BUILD_IN_SOURCE TRUE + # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example + BUILD_COMMAND "$(MAKE)" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c08f894fe7..074014b0d7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -41,6 +41,12 @@ if(${BUILD_WITH_CRAFT}) SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE) endif(${BUILD_WITH_CRAFT}) +# traft +if(${BUILD_WITH_TRAFT}) + cat("${CMAKE_SUPPORT_DIR}/traft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + SET(BUILD_WITH_UV ON CACHE BOOL "traft need libuv" FORCE) +endif(${BUILD_WITH_TRAFT}) + #libuv if(${BUILD_WITH_UV}) cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -173,6 +179,18 @@ if(${BUILD_WITH_CRAFT}) # ) endif(${BUILD_WITH_CRAFT}) +# TRAFT +if(${BUILD_WITH_TRAFT}) + add_library(traft STATIC IMPORTED GLOBAL) + set_target_properties(traft PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/traft/.libs/libraft.a" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/traft/include" + ) + # target_link_libraries(craft + # INTERFACE pthread + # ) +endif(${BUILD_WITH_TRAFT}) + # LIBUV if(${BUILD_WITH_UV}) add_subdirectory(libuv) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 330fe8f70f..0c71113056 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -19,4 +19,8 @@ if(${BUILD_WITH_CRAFT}) add_subdirectory(craft) endif(${BUILD_WITH_CRAFT}) +if(${BUILD_WITH_TRAFT}) + add_subdirectory(traft) +endif(${BUILD_WITH_TRAFT}) + add_subdirectory(tdev) diff --git a/contrib/test/craft/common.h b/contrib/test/craft/common.h index 1e94ee8bca..282e282543 100644 --- a/contrib/test/craft/common.h +++ b/contrib/test/craft/common.h @@ -20,6 +20,7 @@ typedef struct { } Addr; typedef struct { + int voter; Addr me; Addr peers[MAX_PEERS]; int peersCount; diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c index 52e0b694dc..bae083cf94 100644 --- a/contrib/test/craft/raftMain.c +++ b/contrib/test/craft/raftMain.c @@ -104,7 +104,7 @@ const char* state2String(unsigned short state) { void printRaftConfiguration(struct raft_configuration *c) { printf("configuration: \n"); for (int i = 0; i < c->n; ++i) { - printf("%llu -- %d -- %s\n", c->servers->id, c->servers->role, c->servers->address); + printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); } } @@ -119,11 +119,9 @@ void printRaftState(struct raft *r) { printf("last_applied: %llu \n", r->last_applied); printf("last_stored: %llu \n", r->last_stored); - /* printf("configuration_index: %llu \n", r->configuration_index); printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); printRaftConfiguration(&r->configuration); - */ printf("----------------------------\n"); } @@ -164,6 +162,18 @@ void getValue(const char *key) { } } +void raft_change_cb_add(struct raft_change *req, int status) { + printf("raft_change_cb_add status:%d ... \n", status); +} + +void raft_change_cb_assign(struct raft_change *req, int status) { + printf("raft_change_cb_assign status:%d ... \n", status); +} + +void raft_change_cb_remove(struct raft_change *req, int status) { + printf("raft_change_cb_remove status:%d ... \n", status); +} + void console(SRaftServer *pRaftServer) { while (1) { char cmd_buf[COMMAND_LEN]; @@ -193,30 +203,59 @@ void console(SRaftServer *pRaftServer) { parseCommand(cmd_buf, cmd, param1, param2, TOKEN_LEN); if (strcmp(cmd, "addnode") == 0) { - printf("not support \n"); + //printf("not support \n"); - /* char host[HOST_LEN]; uint32_t port; parseAddr(param1, host, HOST_LEN, &port); uint64_t rid = raftId(host, port); struct raft_change *req = raft_malloc(sizeof(*req)); - int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL); + int r = raft_add(&pRaftServer->raft, req, rid, param1, raft_change_cb_add); if (r != 0) { - printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft)); + printf("raft_add error: %s \n", raft_errmsg(&pRaftServer->raft)); } printf("add node: %lu %s \n", rid, param1); struct raft_change *req2 = raft_malloc(sizeof(*req2)); - r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL); + r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, raft_change_cb_assign); if (r != 0) { - printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft)); + printf("raft_assign error: %s \n", raft_errmsg(&pRaftServer->raft)); } - */ + printf("raft_assign: %s %d \n", param1, RAFT_VOTER); + + } else if (strcmp(cmd, "activate") == 0) { + char host[HOST_LEN]; + uint32_t port; + parseAddr(param1, host, HOST_LEN, &port); + uint64_t rid = raftId(host, port); + + + struct raft_change *req2 = raft_malloc(sizeof(*req2)); + int r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, raft_change_cb_assign); + if (r != 0) { + printf("raft_assign error: %s \n", raft_errmsg(&pRaftServer->raft)); + } + printf("raft_assign: %s %d \n", param1, RAFT_VOTER); + + + + } else if (strcmp(cmd, "dropnode") == 0) { - printf("not support \n"); + char host[HOST_LEN]; + uint32_t port; + parseAddr(param1, host, HOST_LEN, &port); + uint64_t rid = raftId(host, port); + + struct raft_change *req = raft_malloc(sizeof(*req)); + int r = raft_remove(&pRaftServer->raft, req, rid, raft_change_cb_remove); + if (r != 0) { + printf("raft_remove: %s \n", raft_errmsg(&pRaftServer->raft)); + } + printf("drop node: %lu %s \n", rid, param1); + + } else if (strcmp(cmd, "put") == 0) { char buf[256]; @@ -234,6 +273,7 @@ void console(SRaftServer *pRaftServer) { } else if (strcmp(cmd, "help") == 0) { printf("addnode \"127.0.0.1:8888\" \n"); + printf("activate \"127.0.0.1:8888\" \n"); printf("dropnode \"127.0.0.1:8888\" \n"); printf("put key value \n"); printf("get key \n"); @@ -256,7 +296,9 @@ void *startConsoleFunc(void *param) { // Config --------------------------------- void usage() { printf("\nusage: \n"); - printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10000 --dir=./data --voter \n", exe_name); + printf("%s --me=127.0.0.1:10001 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10002 --dir=./data \n", exe_name); printf("\n"); printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name); printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name); @@ -271,13 +313,15 @@ void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { option_index = 0; static struct option long_options[] = { {"help", no_argument, NULL, 'h'}, + {"voter", no_argument, NULL, 'v'}, {"peers", required_argument, NULL, 'p'}, {"me", required_argument, NULL, 'm'}, {"dir", required_argument, NULL, 'd'}, {NULL, 0, NULL, 0} }; - while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) { + pConf->voter = 0; + while ((option_value = getopt_long(argc, argv, "hvp:m:d:", long_options, &option_index)) != -1) { switch (option_value) { case 'm': { parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); @@ -295,6 +339,10 @@ void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { break; } + case 'v': { + pConf->voter = 1; + break; + } case 'd': { snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg); @@ -338,6 +386,8 @@ int main(int argc, char **argv) { exit(-1); } + signal(SIGPIPE, SIG_IGN); + SRaftServerConfig conf; parseConf(argc, argv, &conf); printConf(&conf); diff --git a/contrib/test/craft/raftServer.c b/contrib/test/craft/raftServer.c index 6f4dbc1997..ffec22e646 100644 --- a/contrib/test/craft/raftServer.c +++ b/contrib/test/craft/raftServer.c @@ -85,29 +85,45 @@ int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, pRaftServer->fsm = pFsm; ret = uv_loop_init(&pRaftServer->loop); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } ret = raft_uv_init(&pRaftServer->io, &pRaftServer->loop, pRaftServer->dir, &pRaftServer->transport); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } ret = raft_init(&pRaftServer->raft, &pRaftServer->io, pRaftServer->fsm, pRaftServer->raftId, pRaftServer->address); - if (!ret) { + if (ret != 0) { fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft)); + assert(0); } struct raft_configuration conf; raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_VOTER); + + if (pConf->voter == 0) { + raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_SPARE); + + } else { + raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_VOTER); + + } + + + printf("add myself: %llu - %s \n", pRaftServer->raftId, pRaftServer->address); + + for (int i = 0; i < pConf->peersCount; ++i) { const Addr *pAddr = &pConf->peers[i]; raft_id rid = raftId(pAddr->host, pAddr->port); diff --git a/contrib/test/traft/CMakeLists.txt b/contrib/test/traft/CMakeLists.txt new file mode 100644 index 0000000000..e29fea04f1 --- /dev/null +++ b/contrib/test/traft/CMakeLists.txt @@ -0,0 +1,7 @@ +add_executable(raftMain "") +target_sources(raftMain + PRIVATE + "raftMain.c" + "raftServer.c" +) +target_link_libraries(raftMain PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/clear.sh b/contrib/test/traft/clear.sh new file mode 100644 index 0000000000..398b3088f2 --- /dev/null +++ b/contrib/test/traft/clear.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -rf 127.0.0.1* +rm -rf ./data diff --git a/contrib/test/traft/common.h b/contrib/test/traft/common.h new file mode 100644 index 0000000000..0229c29cf7 --- /dev/null +++ b/contrib/test/traft/common.h @@ -0,0 +1,36 @@ +#ifndef TDENGINE_COMMON_H +#define TDENGINE_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define MAX_INSTANCE_NUM 100 + +#define MAX_PEERS 10 +#define COMMAND_LEN 1024 +#define TOKEN_LEN 128 +#define DIR_LEN 256 +#define HOST_LEN 64 +#define ADDRESS_LEN (HOST_LEN + 16) + +typedef struct { + char host[HOST_LEN]; + uint32_t port; +} Addr; + +typedef struct { + Addr me; + Addr peers[MAX_PEERS]; + int peersCount; + char dir[DIR_LEN]; + char dataDir[DIR_LEN + HOST_LEN * 2]; +} SRaftServerConfig; + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_COMMON_H diff --git a/contrib/test/traft/help.txt b/contrib/test/traft/help.txt new file mode 100644 index 0000000000..7709e80e0a --- /dev/null +++ b/contrib/test/traft/help.txt @@ -0,0 +1,18 @@ + +make raftServer + +all: + gcc raftMain.c raftServer.c -I ../../traft/include/ ../../traft/.libs/libraft.a -o raftMain -luv -llz4 -lpthread -g +clean: + rm -f raftMain + sh clear.sh + + +make traft: + +sudo apt-get install libuv1-dev liblz4-dev +autoreconf -i +./configure --enable-example +make + + diff --git a/contrib/test/traft/raftMain.c b/contrib/test/traft/raftMain.c new file mode 100644 index 0000000000..24ad93856c --- /dev/null +++ b/contrib/test/traft/raftMain.c @@ -0,0 +1,659 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "raftServer.h" +#include "common.h" + +const char *exe_name; + +typedef struct LeaderState { + char address[48]; + int leaderCount; + +} LeaderState; + +#define NODE_COUNT 3 +LeaderState leaderStates[NODE_COUNT]; + +void printLeaderCount() { + for (int i = 0; i < NODE_COUNT; ++i) { + printf("%s: leaderCount:%d \n", leaderStates[i].address, leaderStates[i].leaderCount); + } +} + +void updateLeaderStates(SRaftServer *pRaftServer) { + for (int i = 0; i < pRaftServer->instance[0].raft.configuration.n; ++i) { + snprintf(leaderStates[i].address, sizeof(leaderStates[i].address), "%s", pRaftServer->instance[0].raft.configuration.servers[i].address); + leaderStates[i].leaderCount = 0; + } + + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + struct raft *r = &pRaftServer->instance[i].raft; + + char leaderAddress[128]; + memset(leaderAddress, 0, sizeof(leaderAddress)); + + if (r->state == RAFT_LEADER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); + } else if (r->state == RAFT_FOLLOWER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); + } + + for (int j = 0; j < NODE_COUNT; j++) { + if (strcmp(leaderAddress, leaderStates[j].address) == 0) { + leaderStates[j].leaderCount++; + } + } + } +} + + +void raftTransferCb(struct raft_transfer *req) { + SRaftServer *pRaftServer = req->data; + raft_free(req); + + printf("raftTransferCb: \n"); + updateLeaderStates(pRaftServer); + printLeaderCount(); + + int myLeaderCount; + for (int i = 0; i < NODE_COUNT; ++i) { + if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { + myLeaderCount = leaderStates[i].leaderCount; + } + } + + printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); + if (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { + struct raft *r; + for (int j = 0; j < pRaftServer->instanceCount; ++j) { + if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { + r = &pRaftServer->instance[j].raft; + } + } + + struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); + transfer->data = pRaftServer; + + uint64_t destRaftId; + int minIndex = -1; + int minLeaderCount = myLeaderCount; + for (int j = 0; j < NODE_COUNT; ++j) { + if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; + if (leaderStates[j].leaderCount <= minLeaderCount) { + minIndex = j; + } + } + + char myHost[48]; + uint16_t myPort; + uint16_t myVid; + decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); + + char *destAddress = leaderStates[minIndex].address; + + char tokens[MAX_PEERS][MAX_TOKEN_LEN]; + splitString(destAddress, ":", tokens, 2); + char *destHost = tokens[0]; + uint16_t destPort = atoi(tokens[1]); + destRaftId = encodeRaftId(destHost, destPort, myVid); + + raft_transfer(r, transfer, destRaftId, raftTransferCb); + } + +} + + +void parseAddr(const char *addr, char *host, int len, uint32_t *port) { + char* tmp = (char*)malloc(strlen(addr) + 1); + strcpy(tmp, addr); + + char* context; + char* separator = ":"; + char* token = strtok_r(tmp, separator, &context); + if (token) { + snprintf(host, len, "%s", token); + } + + token = strtok_r(NULL, separator, &context); + if (token) { + sscanf(token, "%u", port); + } + + free(tmp); +} + +// only parse 3 tokens +int parseCommand3(const char* str, char* token1, char* token2, char* token3, int len) +{ + char* tmp = (char*)malloc(strlen(str) + 1); + strcpy(tmp, str); + + char* context; + char* separator = " "; + int n = 0; + + char* token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token1, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token2, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token3, token, len); + n++; + } + +ret: + return n; + free(tmp); +} + +// only parse 4 tokens +int parseCommand4(const char* str, char* token1, char* token2, char* token3, char *token4, int len) +{ + char* tmp = (char*)malloc(strlen(str) + 1); + strcpy(tmp, str); + + char* context; + char* separator = " "; + int n = 0; + + char* token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token1, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token2, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token3, token, len); + n++; + } + + token = strtok_r(NULL, separator, &context); + if (!token) { + goto ret; + } + if (strcmp(token, "") != 0) { + strncpy(token4, token, len); + n++; + } + +ret: + return n; + free(tmp); +} + +void *startServerFunc(void *param) { + SRaftServer *pServer = (SRaftServer*)param; + int32_t r = raftServerStart(pServer); + assert(r == 0); + + return NULL; +} + +// Console --------------------------------- +const char* state2String(unsigned short state) { + if (state == RAFT_UNAVAILABLE) { + return "RAFT_UNAVAILABLE"; + + } else if (state == RAFT_FOLLOWER) { + return "RAFT_FOLLOWER"; + + } else if (state == RAFT_CANDIDATE) { + return "RAFT_CANDIDATE"; + + } else if (state == RAFT_LEADER) { + return "RAFT_LEADER"; + + } + return "UNKNOWN_RAFT_STATE"; +} + + +void printRaftState2(struct raft *r) { + + char leaderAddress[128]; + memset(leaderAddress, 0, sizeof(leaderAddress)); + + if (r->state == RAFT_LEADER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); + } else if (r->state == RAFT_FOLLOWER) { + snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); + } + + for (int i = 0; i < r->configuration.n; ++i) { + char tmpAddress[128]; + snprintf(tmpAddress, sizeof(tmpAddress), "%s", r->configuration.servers[i].address); + + uint64_t raftId = r->configuration.servers[i].id; + char host[128]; + uint16_t port; + uint16_t vid; + decodeRaftId(raftId, host, 128, &port, &vid); + + char buf[512]; + memset(buf, 0, sizeof(buf)); + if (strcmp(tmpAddress, leaderAddress) == 0) { + snprintf(buf, sizeof(buf), "<%s:%u-%u-LEADER>\t", host, port, vid); + } else { + snprintf(buf, sizeof(buf), "<%s:%u-%u-FOLLOWER>\t", host, port, vid); + } + printf("%s", buf); + } + printf("\n"); +} + +void printRaftConfiguration(struct raft_configuration *c) { + printf("configuration: \n"); + for (int i = 0; i < c->n; ++i) { + printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); + } +} + +void printRaftState(struct raft *r) { + printf("----Raft State: -----------\n"); + printf("mem_addr: %p \n", r); + printf("my_id: %llu \n", r->id); + printf("address: %s \n", r->address); + printf("current_term: %llu \n", r->current_term); + printf("voted_for: %llu \n", r->voted_for); + printf("role: %s \n", state2String(r->state)); + printf("commit_index: %llu \n", r->commit_index); + printf("last_applied: %llu \n", r->last_applied); + printf("last_stored: %llu \n", r->last_stored); + + printf("configuration_index: %llu \n", r->configuration_index); + printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); + printRaftConfiguration(&r->configuration); + + printf("----------------------------\n"); +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + raft_free(req); + struct raft *r = req->data; + if (status != 0) { + printf("putValueCb: %s \n", raft_errmsg(r)); + } else { + printf("putValueCb: %s \n", "ok"); + } +} + +void putValue(struct raft *r, const char *value) { + struct raft_buffer buf; + + buf.len = TOKEN_LEN;; + buf.base = raft_malloc(buf.len); + snprintf(buf.base, buf.len, "%s", value); + + struct raft_apply *req = raft_malloc(sizeof(struct raft_apply)); + req->data = r; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char*)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +void getValue(const char *key) { + char *ptr = getKV(key); + if (ptr) { + printf("get value: [%s] \n", ptr); + } else { + printf("value not found for key: [%s] \n", key); + } +} + +void console(SRaftServer *pRaftServer) { + while (1) { + char cmd_buf[COMMAND_LEN]; + memset(cmd_buf, 0, sizeof(cmd_buf)); + char *ret = fgets(cmd_buf, COMMAND_LEN, stdin); + if (!ret) { + exit(-1); + } + + int pos = strlen(cmd_buf); + if(cmd_buf[pos - 1] == '\n') { + cmd_buf[pos - 1] = '\0'; + } + + if (strncmp(cmd_buf, "", COMMAND_LEN) == 0) { + continue; + } + + char cmd[TOKEN_LEN]; + memset(cmd, 0, sizeof(cmd)); + + char param1[TOKEN_LEN]; + memset(param1, 0, sizeof(param1)); + + char param2[TOKEN_LEN]; + memset(param2, 0, sizeof(param2)); + + char param3[TOKEN_LEN]; + memset(param2, 0, sizeof(param2)); + + parseCommand4(cmd_buf, cmd, param1, param2, param3, TOKEN_LEN); + if (strcmp(cmd, "addnode") == 0) { + printf("not support \n"); + + /* + char host[HOST_LEN]; + uint32_t port; + parseAddr(param1, host, HOST_LEN, &port); + uint64_t rid = raftId(host, port); + + struct raft_change *req = raft_malloc(sizeof(*req)); + int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL); + if (r != 0) { + printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft)); + } + printf("add node: %lu %s \n", rid, param1); + + struct raft_change *req2 = raft_malloc(sizeof(*req2)); + r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL); + if (r != 0) { + printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft)); + } + */ + + } else if (strcmp(cmd, "dropnode") == 0) { + printf("not support \n"); + + } else if (strcmp(cmd, "rebalance") == 0) { + + /* + updateLeaderStates(pRaftServer); + + int myLeaderCount; + for (int i = 0; i < NODE_COUNT; ++i) { + if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { + myLeaderCount = leaderStates[i].leaderCount; + } + } + + while (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { + printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); + + struct raft *r; + for (int j = 0; j < pRaftServer->instanceCount; ++j) { + if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { + r = &pRaftServer->instance[j].raft; + } + } + + struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); + transfer->data = pRaftServer; + + uint64_t destRaftId; + int minIndex = -1; + int minLeaderCount = myLeaderCount; + for (int j = 0; j < NODE_COUNT; ++j) { + if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; + + printf("-----leaderStates[%d].leaderCount:%d \n", j, leaderStates[j].leaderCount); + if (leaderStates[j].leaderCount <= minLeaderCount) { + minIndex = j; + printf("++++ assign minIndex : %d \n", minIndex); + } + } + + printf("minIndex:%d minLeaderCount:%d \n", minIndex, minLeaderCount); + + char myHost[48]; + uint16_t myPort; + uint16_t myVid; + decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); + + char *destAddress = leaderStates[minIndex].address; + + char tokens[MAX_PEERS][MAX_TOKEN_LEN]; + splitString(destAddress, ":", tokens, 2); + char *destHost = tokens[0]; + uint16_t destPort = atoi(tokens[1]); + destRaftId = encodeRaftId(destHost, destPort, myVid); + + printf("destHost:%s destPort:%u myVid:%u", destHost, destPort, myVid); + raft_transfer(r, transfer, destRaftId, raftTransferCb); + sleep(1); + + for (int i = 0; i < NODE_COUNT; ++i) { + if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { + myLeaderCount = leaderStates[i].leaderCount; + } + } + } + */ + + + int leaderCount = 0; + + struct raft *firstR; + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + struct raft *r = &pRaftServer->instance[i].raft; + if (r->state == RAFT_LEADER) { + leaderCount++; + firstR = r; + } + } + + if (leaderCount > pRaftServer->instanceCount / NODE_COUNT) { + struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); + transfer->data = pRaftServer; + raft_transfer(firstR, transfer, 0, raftTransferCb); + } + + + } else if (strcmp(cmd, "put") == 0) { + char buf[256]; + uint16_t vid; + sscanf(param1, "%hu", &vid); + snprintf(buf, sizeof(buf), "%s--%s", param2, param3); + putValue(&pRaftServer->instance[vid].raft, buf); + + } else if (strcmp(cmd, "get") == 0) { + getValue(param1); + + } else if (strcmp(cmd, "transfer") == 0) { + uint16_t vid; + sscanf(param1, "%hu", &vid); + + struct raft_transfer transfer; + raft_transfer(&pRaftServer->instance[vid].raft, &transfer, 0, NULL); + + + } else if (strcmp(cmd, "state") == 0) { + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + printf("instance %d: ", i); + printRaftState(&pRaftServer->instance[i].raft); + } + + } else if (strcmp(cmd, "state2") == 0) { + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + printRaftState2(&pRaftServer->instance[i].raft); + } + + } else if (strcmp(cmd, "snapshot") == 0) { + printf("not support \n"); + + } else if (strcmp(cmd, "help") == 0) { + printf("addnode \"127.0.0.1:8888\" \n"); + printf("dropnode \"127.0.0.1:8888\" \n"); + printf("put key value \n"); + printf("get key \n"); + printf("state \n"); + + } else { + printf("unknown command: [%s], type \"help\" to see help \n", cmd); + } + + //printf("cmd_buf: [%s] \n", cmd_buf); + } +} + +void *startConsoleFunc(void *param) { + SRaftServer *pServer = (SRaftServer*)param; + console(pServer); + return NULL; +} + +// Config --------------------------------- +void usage() { + printf("\nusage: \n"); + printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name); + printf("\n"); + printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name); + printf("%s --me=127.0.0.1:10002 --peers=127.0.0.1:10000,127.0.0.1:10001 --dir=./data \n", exe_name); + printf("\n"); +} + +void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { + memset(pConf, 0, sizeof(*pConf)); + + int option_index, option_value; + option_index = 0; + static struct option long_options[] = { + {"help", no_argument, NULL, 'h'}, + {"peers", required_argument, NULL, 'p'}, + {"me", required_argument, NULL, 'm'}, + {"dir", required_argument, NULL, 'd'}, + {NULL, 0, NULL, 0} + }; + + while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) { + switch (option_value) { + case 'm': { + parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); + break; + } + + case 'p': { + char tokens[MAX_PEERS][MAX_TOKEN_LEN]; + int peerCount = splitString(optarg, ",", tokens, MAX_PEERS); + pConf->peersCount = peerCount; + for (int i = 0; i < peerCount; ++i) { + Addr *pAddr = &pConf->peers[i]; + parseAddr(tokens[i], pAddr->host, sizeof(pAddr->host), &pAddr->port); + } + break; + } + + + case 'd': { + snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg); + break; + } + + case 'h': { + usage(); + exit(-1); + } + + default: { + usage(); + exit(-1); + } + } + } + snprintf(pConf->dataDir, sizeof(pConf->dataDir), "%s/%s_%u", pConf->dir, pConf->me.host, pConf->me.port); +} + +void printConf(SRaftServerConfig *pConf) { + printf("\nconf: \n"); + printf("me: %s:%u \n", pConf->me.host, pConf->me.port); + printf("peersCount: %d \n", pConf->peersCount); + for (int i = 0; i < pConf->peersCount; ++i) { + Addr *pAddr = &pConf->peers[i]; + printf("peer%d: %s:%u \n", i, pAddr->host, pAddr->port); + } + printf("dataDir: %s \n\n", pConf->dataDir); + +} + + +int main(int argc, char **argv) { + srand(time(NULL)); + int32_t ret; + + exe_name = argv[0]; + if (argc < 3) { + usage(); + exit(-1); + } + + SRaftServerConfig conf; + parseConf(argc, argv, &conf); + printConf(&conf); + + signal(SIGPIPE, SIG_IGN); + + /* + char cmd_buf[COMMAND_LEN]; + snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", conf.dataDir); + system(cmd_buf); + */ + + + struct raft_fsm fsm; + initFsm(&fsm); + + SRaftServer raftServer; + ret = raftServerInit(&raftServer, &conf, &fsm); + assert(ret == 0); + + pthread_t tidRaftServer; + pthread_create(&tidRaftServer, NULL, startServerFunc, &raftServer); + + pthread_t tidConsole; + pthread_create(&tidConsole, NULL, startConsoleFunc, &raftServer); + + while (1) { + sleep(10); + } + + return 0; +} diff --git a/contrib/test/traft/raftServer.c b/contrib/test/traft/raftServer.c new file mode 100644 index 0000000000..94de49cd0f --- /dev/null +++ b/contrib/test/traft/raftServer.c @@ -0,0 +1,222 @@ +#include +#include +#include "common.h" +#include "raftServer.h" + +char *keys; +char *values; + +void initStore() { + keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); + values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); + writeIndex = 0; +} + +void destroyStore() { + free(keys); + free(values); +} + +void putKV(const char *key, const char *value) { + if (writeIndex < MAX_RECORD_COUNT) { + strncpy(&keys[writeIndex], key, MAX_KV_LEN); + strncpy(&values[writeIndex], value, MAX_KV_LEN); + writeIndex++; + } +} + +char *getKV(const char *key) { + for (int i = 0; i < MAX_RECORD_COUNT; ++i) { + if (strcmp(&keys[i], key) == 0) { + return &values[i]; + } + } + return NULL; +} + + +int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr) +{ + if (n_arr <= 0) { + return -1; + } + + char* tmp = (char*)malloc(strlen(str) + 1); + strcpy(tmp, str); + char* context; + int n = 0; + + char* token = strtok_r(tmp, separator, &context); + if (!token) { + goto ret; + } + strncpy(arr[n], token, MAX_TOKEN_LEN); + n++; + + while (1) { + token = strtok_r(NULL, separator, &context); + if (!token || n >= n_arr) { + goto ret; + } + strncpy(arr[n], token, MAX_TOKEN_LEN); + n++; + } + +ret: + free(tmp); + return n; +} + +/* +uint64_t raftId(const char *host, uint32_t port) { + uint32_t host_uint32 = (uint32_t)inet_addr(host); + assert(host_uint32 != (uint32_t)-1); + uint64_t code = ((uint64_t)host_uint32) << 32 | port; + return code; +} +*/ + + +/* +uint64_t encodeRaftId(const char *host, uint16_t port, uint16_t vid) { + uint64_t raftId; + uint32_t host_uint32 = (uint32_t)inet_addr(host); + assert(host_uint32 != (uint32_t)-1); + + raftId = (((uint64_t)host_uint32) << 32) | (((uint32_t)port) << 16) | vid; + return raftId; +} + +void decodeRaftId(uint64_t raftId, char *host, int32_t len, uint16_t *port, uint16_t *vid) { + uint32_t host32 = (uint32_t)((raftId >> 32) & 0x00000000FFFFFFFF); + + struct in_addr addr; + addr.s_addr = host32; + snprintf(host, len, "%s", inet_ntoa(addr)); + + *port = (uint16_t)((raftId & 0x00000000FFFF0000) >> 16); + *vid = (uint16_t)(raftId & 0x000000000000FFFF); +} +*/ + + + + +int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm) { + int ret; + + snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); + pRaftServer->port = pConf->me.port; + snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); + //strncpy(pRaftServer->dir, pConf->dataDir, sizeof(pRaftServer->dir)); + + ret = uv_loop_init(&pRaftServer->loop); + if (ret != 0) { + fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); + assert(0); + } + + ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); + if (ret != 0) { + fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); + assert(0); + } + + + uint16_t vid; + pRaftServer->instanceCount = 20; + + + for (int i = 0; i < pRaftServer->instanceCount; ++i) + { + //vid = 0; + vid = i; + + + pRaftServer->instance[vid].raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); + snprintf(pRaftServer->instance[vid].dir, sizeof(pRaftServer->instance[vid].dir), "%s_%llu", pConf->dataDir, pRaftServer->instance[vid].raftId); + + char cmd_buf[COMMAND_LEN]; + snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", pRaftServer->instance[vid].dir); + system(cmd_buf); + sleep(1); + + pRaftServer->instance[vid].fsm = pFsm; + + ret = raft_uv_init(&pRaftServer->instance[vid].io, &pRaftServer->loop, pRaftServer->instance[vid].dir, &pRaftServer->transport); + if (ret != 0) { + fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); + assert(0); + } + + ret = raft_init(&pRaftServer->instance[vid].raft, &pRaftServer->instance[vid].io, pRaftServer->instance[vid].fsm, pRaftServer->instance[vid].raftId, pRaftServer->address); + if (ret != 0) { + fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); + assert(0); + } + + struct raft_configuration conf; + raft_configuration_init(&conf); + raft_configuration_add(&conf, pRaftServer->instance[vid].raftId, pRaftServer->address, RAFT_VOTER); + printf("add myself: %llu - %s \n", pRaftServer->instance[vid].raftId, pRaftServer->address); + for (int i = 0; i < pConf->peersCount; ++i) { + const Addr *pAddr = &pConf->peers[i]; + + raft_id rid = encodeRaftId(pAddr->host, pAddr->port, vid); + + char addrBuf[ADDRESS_LEN]; + snprintf(addrBuf, sizeof(addrBuf), "%s:%u", pAddr->host, pAddr->port); + raft_configuration_add(&conf, rid, addrBuf, RAFT_VOTER); + printf("add peers: %llu - %s \n", rid, addrBuf); + } + + raft_bootstrap(&pRaftServer->instance[vid].raft, &conf); + + } + + + + + + + + return 0; +} + +int32_t raftServerStart(SRaftServer *pRaftServer) { + int ret; + + for (int i = 0; i < pRaftServer->instanceCount; ++i) { + ret = raft_start(&pRaftServer->instance[i].raft); + if (ret != 0) { + fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[i].raft)); + } + + } + + + uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); +} + + +void raftServerClose(SRaftServer *pRaftServer) { + +} + + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { + char *msg = (char*)buf->base; + printf("fsm apply: %s \n", msg); + + char arr[2][MAX_TOKEN_LEN]; + splitString(msg, "--", arr, 2); + putKV(arr[0], arr[1]); + + return 0; +} + +int32_t initFsm(struct raft_fsm *fsm) { + initStore(); + fsm->apply = fsmApplyCb; + return 0; +} diff --git a/contrib/test/traft/raftServer.h b/contrib/test/traft/raftServer.h new file mode 100644 index 0000000000..b1f62caac5 --- /dev/null +++ b/contrib/test/traft/raftServer.h @@ -0,0 +1,68 @@ +#ifndef TDENGINE_RAFT_SERVER_H +#define TDENGINE_RAFT_SERVER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include "raft.h" +#include "raft/uv.h" +#include "common.h" + + +// simulate a db store, just for test +#define MAX_KV_LEN 100 +#define MAX_RECORD_COUNT 500 +char *keys; +char *values; +int writeIndex; + +void initStore(); +void destroyStore(); +void putKV(const char *key, const char *value); +char *getKV(const char *key); + +typedef struct { + char dir[DIR_LEN + HOST_LEN * 4]; /* Data dir of UV I/O backend */ + raft_id raftId; /* For vote */ + struct raft_fsm *fsm; /* Sample application FSM */ + struct raft raft; /* Raft instance */ + struct raft_io io; /* UV I/O backend */ + +} SInstance; + +typedef struct { + char host[HOST_LEN]; + uint32_t port; + char address[ADDRESS_LEN]; /* Raft instance address */ + + struct uv_loop_s loop; /* UV loop */ + struct raft_uv_transport transport; /* UV I/O backend transport */ + + SInstance instance[MAX_INSTANCE_NUM]; + int32_t instanceCount; + +} SRaftServer; + +#define MAX_TOKEN_LEN 32 +int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr); + +int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm); +int32_t raftServerStart(SRaftServer *pRaftServer); +void raftServerClose(SRaftServer *pRaftServer); + + +int initFsm(struct raft_fsm *fsm); + + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_RAFT_SERVER_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 42956b6bdd..e8a56b77c9 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -30,11 +30,12 @@ extern char tsLocalEp[]; extern uint16_t tsServerPort; extern int32_t tsStatusInterval; extern int8_t tsEnableTelemetryReporting; +extern int32_t tsNumOfSupportVnodes; // common extern int tsRpcTimer; extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled +extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; @@ -48,14 +49,18 @@ extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; extern char tsTempDir[]; -extern int64_t tsMaxVnodeQueuedBytes; -extern int tsCompatibleModel; // 2.0 compatible model +extern int tsCompatibleModel; // 2.0 compatible model +extern int8_t tsEnableSlaveQuery; +extern int8_t tsEnableAdjustMaster; +extern int8_t tsPrintAuth; +extern int64_t tsTickPerDay[3]; -//query buffer management -extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing -extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing -extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked -extern int8_t tsKeepOriginalColumnName; +// query buffer management +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing +extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node +extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked +extern int8_t tsKeepOriginalColumnName; +extern int8_t tsDeadLockKillQuery; // client extern int32_t tsMaxSQLStringLen; @@ -72,27 +77,17 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// balance -extern int8_t tsEnableSlaveQuery; - - -// interna -extern int8_t tsPrintAuth; -extern char tsVnodeDir[]; -extern char tsMnodeDir[]; -extern int64_t tsTickPerDay[3]; - // system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; +extern float tsTotalLogDirGB; +extern float tsTotalTmpDirGB; +extern float tsTotalDataDirGB; +extern float tsAvailLogDirGB; +extern float tsAvailTmpDirectorySpace; +extern float tsAvailDataDirGB; +extern float tsUsedDataDirGB; +extern float tsMinimalLogDirGB; +extern float tsReservedTmpDirectorySpace; +extern float tsMinimalDataDirGB; extern uint32_t tsVersion; // build info @@ -102,17 +97,13 @@ extern char gitinfo[]; extern char gitinfoOfInternal[]; extern char buildinfo[]; -#ifdef TD_TSZ -// lossy -extern char lossyColumns[]; -extern double fPrecision; -extern double dPrecision; -extern uint32_t maxRange; -extern uint32_t curRange; -extern char Compressor[]; -#endif -// long query -extern int8_t tsDeadLockKillQuery; +// lossy +extern char tsLossyColumns[]; +extern double tsFPrecision; +extern double tsDPrecision; +extern uint32_t tsMaxRange; +extern uint32_t tsCurRange; +extern char tsCompressor[]; typedef struct { char dir[TSDB_FILENAME_LEN]; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b9b1d3fe76..17a4d4ad2c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -25,7 +25,9 @@ extern "C" { #include "taoserror.h" #include "tcoding.h" #include "tdataformat.h" +#include "tlist.h" +/* ------------------------ MESSAGE DEFINITIONS ------------------------ */ #define TD_MSG_NUMBER_ #undef TD_MSG_DICT_ #undef TD_MSG_INFO_ @@ -54,6 +56,47 @@ extern int tMsgDict[]; typedef uint16_t tmsg_t; +/* ------------------------ ENCODE/DECODE FUNCTIONS AND MACROS ------------------------ */ +struct SMEListNode { + TD_SLIST_NODE(SMEListNode); + SEncoder coder; +}; + +typedef struct SMsgEncoder { + SEncoder coder; + TD_SLIST(SMEListNode) eStack; // encode stack +} SMsgEncoder; + +struct SMDFreeListNode { + TD_SLIST_NODE(SMDFreeListNode); + char payload[]; +}; + +struct SMDListNode { + TD_SLIST_NODE(SMDListNode); + SDecoder coder; +}; + +typedef struct SMsgDecoder { + SDecoder coder; + TD_SLIST(SMDListNode) dStack; + TD_SLIST(SMDFreeListNode) freeList; +} SMsgDecoder; + +#define TMSG_MALLOC(SIZE, DECODER) \ + ({ \ + void* ptr = malloc((SIZE) + sizeof(struct SMDFreeListNode)); \ + if (ptr) { \ + TD_SLIST_PUSH(&((DECODER)->freeList), (struct SMDFreeListNode*)ptr); \ + ptr = POINTER_SHIFT(ptr, sizeof(struct SMDFreeListNode*)); \ + } \ + ptr; \ + }) + +void tmsgInitMsgDecoder(SMsgDecoder* pMD, td_endian_t endian, uint8_t* data, int64_t size); +void tmsgClearMsgDecoder(SMsgDecoder* pMD); + +/* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 #define TSDB_IE_TYPE_META 2 @@ -71,6 +114,9 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_TABLE, TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, + TSDB_MGMT_TABLE_QNODE, + TSDB_MGMT_TABLE_SNODE, + TSDB_MGMT_TABLE_BNODE, TSDB_MGMT_TABLE_VGROUP, TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_MODULE, @@ -219,6 +265,26 @@ typedef struct { char data[]; } SMDCreateTableMsg; +// typedef struct { +// int32_t len; // one create table message +// char tableName[TSDB_TABLE_FNAME_LEN]; +// int16_t numOfColumns; +// int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string +// int8_t igExists; +// int8_t rspMeta; +// int8_t reserved[16]; +// char schema[]; +//} SCreateTableMsg; + +typedef struct { + char tableName[TSDB_TABLE_FNAME_LEN]; + int16_t numOfColumns; + int16_t numOfTags; + int8_t igExists; + int8_t rspMeta; + char schema[]; +} SCreateCTableMsg; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -242,19 +308,7 @@ typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; uint64_t suid; - int32_t sverson; - uint32_t ttl; - uint32_t keep; - int32_t numOfTags; - int32_t numOfColumns; - SSchema pSchema[]; -} SCreateStbInternalMsg; - -typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - uint64_t suid; -} SDropStbInternalMsg; +} SVDropStbReq; typedef struct { SMsgHead head; @@ -299,6 +353,18 @@ typedef struct SEpSet { char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SEpSet; +static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { + if(buf == NULL) return sizeof(SEpSet); + memcpy(buf, pEp, sizeof(SEpSet)); + //TODO: endian conversion + return sizeof(SEpSet); +} + +static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEpSet) { + memcpy(pEpSet, buf, sizeof(SEpSet)); + return buf; +} + typedef struct { int32_t acctId; int64_t clusterId; @@ -480,7 +546,7 @@ typedef struct { typedef struct { SMsgHead header; union { - int32_t showId; + int64_t showId; int64_t qhandle; int64_t qId; }; // query handle @@ -628,8 +694,8 @@ typedef struct { int64_t clusterId; int64_t rebootTime; int64_t updateTime; - int16_t numOfCores; - int16_t numOfSupportVnodes; + int32_t numOfCores; + int32_t numOfSupportVnodes; char dnodeEp[TSDB_EP_LEN]; SClusterCfg clusterCfg; SVnodeLoads vnodeLoads; @@ -800,7 +866,7 @@ typedef struct { } SCompactMsg; typedef struct SShowRsp { - int32_t showId; + int64_t showId; STableMetaMsg tableMeta; } SShowRsp; @@ -820,29 +886,25 @@ typedef struct { typedef struct { int32_t dnodeId; -} SCreateMnodeMsg, SDropMnodeMsg; +} SMCreateMnodeMsg, SMDropMnodeMsg, SDDropMnodeMsg; typedef struct { int32_t dnodeId; int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; -} SCreateMnodeInMsg, SAlterMnodeInMsg; +} SDCreateMnodeMsg, SDAlterMnodeMsg; typedef struct { int32_t dnodeId; -} SDropMnodeInMsg; +} SMCreateQnodeMsg, SMDropQnodeMsg, SDCreateQnodeMsg, SDDropQnodeMsg; typedef struct { int32_t dnodeId; -} SCreateQnodeInMsg, SDropQnodeInMsg; +} SMCreateSnodeMsg, SMDropSnodeMsg, SDCreateSnodeMsg, SDDropSnodeMsg; typedef struct { int32_t dnodeId; -} SCreateSnodeInMsg, SDropSnodeInMsg; - -typedef struct { - int32_t dnodeId; -} SCreateBnodeInMsg, SDropBnodeInMsg; +} SMCreateBnodeMsg, SMDropBnodeMsg, SDCreateBnodeMsg, SDDropBnodeMsg; typedef struct { int32_t dnodeId; @@ -1056,26 +1118,94 @@ typedef struct STaskDropRsp { } STaskDropRsp; typedef struct { - int8_t igExists; - char* name; - char* phyPlan; + int8_t igExists; + char* name; + char* physicalPlan; + char* logicalPlan; } SCMCreateTopicReq; static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) { int tlen = 0; tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedI8(buf, pReq->igExists); - tlen += taosEncodeString(buf, pReq->phyPlan); + tlen += taosEncodeString(buf, pReq->physicalPlan); + tlen += taosEncodeString(buf, pReq->logicalPlan); return tlen; } static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) { buf = taosDecodeFixedI8(buf, &(pReq->igExists)); buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeString(buf, &(pReq->phyPlan)); + buf = taosDecodeString(buf, &(pReq->physicalPlan)); + buf = taosDecodeString(buf, &(pReq->logicalPlan)); return buf; } +typedef struct { + int64_t topicId; +} SCMCreateTopicRsp; + +static FORCE_INLINE int tSerializeSCMCreateTopicRsp(void** buf, const SCMCreateTopicRsp* pRsp) { + int tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->topicId); + return tlen; +} + +static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopicRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->topicId); + return buf; +} + +typedef struct { + char* topicName; + char* consumerGroup; + int64_t consumerId; +} SCMSubscribeReq; + +static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { + int tlen = 0; + tlen += taosEncodeString(buf, pReq->topicName); + tlen += taosEncodeString(buf, pReq->consumerGroup); + tlen += taosEncodeFixedI64(buf, pReq->consumerId); + return tlen; +} + +static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { + buf = taosDecodeString(buf, &pReq->topicName); + buf = taosDecodeString(buf, &pReq->consumerGroup); + buf = taosDecodeFixedI64(buf, &pReq->consumerId); + return buf; +} + +typedef struct { + int32_t vgId; + SEpSet pEpSet; +} SCMSubscribeRsp; + +static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, pRsp->vgId); + tlen += taosEncodeSEpSet(buf, &pRsp->pEpSet); + return tlen; +} + +static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { + buf = taosDecodeFixedI32(buf, &pRsp->vgId); + buf = taosDecodeSEpSet(buf, &pRsp->pEpSet); + return buf; +} + +typedef struct { + int64_t topicId; + int64_t consumerId; + int64_t consumerGroupId; + int64_t offset; +} SMVSubscribeReq; + +typedef struct { + int64_t newOffset; +} SMVSubscribeRsp; + typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; int8_t igExists; @@ -1105,13 +1235,13 @@ typedef struct { char* executor; int32_t sqlLen; char* sql; -} SCreateTopicInternalMsg; +} SDCreateTopicMsg; typedef struct { SMsgHead head; char name[TSDB_TABLE_FNAME_LEN]; uint64_t tuid; -} SDropTopicInternalMsg; +} SDDropTopicMsg; typedef struct SVCreateTbReq { uint64_t ver; // use a general definition @@ -1141,100 +1271,11 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -static FORCE_INLINE int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq) { - int tlen = 0; +int tmsgSVCreateTbReqEncode(SMsgEncoder* pCoder, SVCreateTbReq* pReq); +int tmsgSVCreateTbReqDecode(SMsgDecoder* pCoder, SVCreateTbReq* pReq); +int tSerializeSVCreateTbReq(void** buf, const SVCreateTbReq* pReq); +void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); - tlen += taosEncodeFixedU64(buf, pReq->ver); - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedU32(buf, pReq->ttl); - tlen += taosEncodeFixedU32(buf, pReq->keep); - tlen += taosEncodeFixedU8(buf, pReq->type); - - switch (pReq->type) { - case TD_SUPER_TABLE: - tlen += taosEncodeFixedU64(buf, pReq->stbCfg.suid); - tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); - for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); - } - tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); - for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); - } - break; - case TD_CHILD_TABLE: - tlen += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); - tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); - break; - case TD_NORMAL_TABLE: - tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); - for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { - tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); - tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); - tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); - } - break; - default: - ASSERT(0); - } - - return tlen; -} - -static FORCE_INLINE void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeFixedU32(buf, &(pReq->ttl)); - buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->type)); - - switch (pReq->type) { - case TD_SUPER_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); - buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); - pReq->stbCfg.pSchema = (SSchema*)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); - buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); - buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); - } - buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); - pReq->stbCfg.pTagSchema = (SSchema*)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); - buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); - } - break; - case TD_CHILD_TABLE: - buf = taosDecodeFixedU64(buf, &pReq->ctbCfg.suid); - buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); - break; - case TD_NORMAL_TABLE: - buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); - pReq->ntbCfg.pSchema = (SSchema*)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); - for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { - buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); - buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); - } - break; - default: - ASSERT(0); - } - - return buf; -} typedef struct SVCreateTbRsp { } SVCreateTbRsp; @@ -1249,7 +1290,7 @@ typedef struct SVShowTablesRsp { typedef struct SVShowTablesFetchReq { SMsgHead head; - int64_t id; + int32_t id; } SVShowTablesFetchReq; typedef struct SVShowTablesFetchRsp { diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2ed817fca1..d6071ebcf3 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -70,6 +70,15 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE, "dnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_QNODE, "dnode-alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_QNODE, "dnode-drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_SNODE, "dnode-create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_SNODE, "dnode-alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_SNODE, "dnode-drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_BNODE, "dnode-alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) @@ -90,9 +99,20 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_DROP_USER, "mnode-drop-user", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DNODE, "mnode-create-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_DNODE, "mnode-config-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DNODE, "mnode-alter-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DNODE, "mnode-drop-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "mnode-create-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_MNODE, "mnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "mnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_QNODE, "mnode-create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "mnode-alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "mnode-drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "mnode-create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "mnode-alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "mnode-drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "mnode-create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "mnode-alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "mnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "mnode-create-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "mnode-drop-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "mnode-use-db", NULL, NULL) @@ -116,9 +136,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SCMCreateTopicReq, SCMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -149,6 +170,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) + TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) + + // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) @@ -158,4 +182,4 @@ enum { #if defined(TD_MSG_NUMBER_) TDMT_MAX #endif -}; \ No newline at end of file +}; diff --git a/include/common/tname.h b/include/common/tname.h index 3ac7f8b27b..11d97dac06 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -17,6 +17,7 @@ #define TDENGINE_TNAME_H #include "tdef.h" +#include "tmsg.h" #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 @@ -58,4 +59,6 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, int32_t acctId); +SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); + #endif // TDENGINE_TNAME_H diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e12ce09422..2e3863c3a1 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -27,8 +27,8 @@ typedef struct SDnode SDnode; typedef struct { int32_t sver; - int16_t numOfCores; - int16_t numOfSupportVnodes; + int32_t numOfCores; + int32_t numOfSupportVnodes; int16_t numOfCommitThreads; int8_t enableTelem; int32_t statusInterval; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 474e526186..3ff86bea3e 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -157,16 +157,21 @@ typedef enum { SDB_TRANS = 1, SDB_CLUSTER = 2, SDB_MNODE = 3, - SDB_DNODE = 4, - SDB_USER = 5, - SDB_AUTH = 6, - SDB_ACCT = 7, - SDB_TOPIC = 8, - SDB_VGROUP = 9, - SDB_STB = 10, - SDB_DB = 11, - SDB_FUNC = 12, - SDB_MAX = 13 + SDB_QNODE = 4, + SDB_SNODE = 5, + SDB_BNODE = 6, + SDB_DNODE = 7, + SDB_USER = 8, + SDB_AUTH = 9, + SDB_ACCT = 10, + SDB_CONSUMER = 11, + SDB_CGROUP = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; @@ -176,6 +181,7 @@ typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); +typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3); typedef struct { ESdbType sdbType; @@ -274,7 +280,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * * @param pSdb The sdb object. * @param type The type of the table. - * @param type The initial iterator of the table. + * @param pIter The initial iterator of the table. * @param pObj The object of the row just fetched. * @return void* The next iterator of the table. */ @@ -284,11 +290,22 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); * @brief Cancel a traversal * * @param pSdb The sdb object. - * @param pIter The iterator of the table. * @param type The initial iterator of table. */ void sdbCancelFetch(SSdb *pSdb, void *pIter); +/** + * @brief Traverse a sdb + * + * @param pSdb The sdb object. + * @param type The initial iterator of table. + * @param fp The function pointer. + * @param p1 The callback param. + * @param p2 The callback param. + * @param p3 The callback param. + */ +void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3); + /** * @brief Get the number of rows in the table * diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index cc5eabf3bf..b75e478add 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -37,6 +37,13 @@ typedef struct SMetaCfg { uint64_t lruSize; } SMetaCfg; +typedef struct { + int32_t nCols; + SSchema *pSchema; +} SSchemaWrapper; + +typedef struct SMTbCursor SMTbCursor; + typedef SVCreateTbReq STbCfg; // SMeta operations @@ -48,7 +55,13 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // For Query -int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg); +STbCfg * metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid); +STbCfg * metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid); +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline); + +SMTbCursor * metaOpenTbCursor(SMeta *pMeta); +void metaCloseTbCursor(SMTbCursor *pTbCur); +char *metaTbCursorNext(SMTbCursor *pTbCur); // Options void metaOptionsInit(SMetaCfg *pMetaCfg); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 8f4b9e1807..1bd29ce396 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -54,11 +54,11 @@ int32_t catalogInit(SCatalogCfg *cfg); /** * Get a cluster's catalog handle for all later operations. - * @param clusterId (input, end with \0) + * @param clusterId * @param catalogHandle (output, NO need to free it) * @return error code */ -int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle); int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 980219a4e9..041adbb582 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -169,6 +169,7 @@ typedef struct SDclStmtInfo { SEpSet epSet; char* pMsg; int32_t msgLen; + void* pExtension; // todo remove it soon } SDclStmtInfo; #ifdef __cplusplus diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index dd3d92866f..a9e1f26d20 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -74,7 +74,6 @@ int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo); STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); -SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); int32_t getNewResColId(); void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 83047a44de..9897467230 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -54,8 +54,11 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); -void qWorkerDestroy(void **qWorkerMgmt); +int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); +int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + +void qWorkerDestroy(void **qWorkerMgmt); #ifdef __cplusplus } diff --git a/include/os/osEnv.h b/include/os/osEnv.h index b857670214..a7fd86776c 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -21,7 +21,6 @@ extern "C" { #endif extern char tsOsName[]; -extern char tsDnodeDir[]; extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; diff --git a/include/util/encode.h b/include/util/encode.h index 85be2b76c6..ee35791012 100644 --- a/include/util/encode.h +++ b/include/util/encode.h @@ -26,8 +26,8 @@ extern "C" { typedef struct { td_endian_t endian; uint8_t* data; - int64_t size; - int64_t pos; + int32_t size; + int32_t pos; } SEncoder, SDecoder; #define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) @@ -62,7 +62,7 @@ typedef struct { #define TD_CHECK_CODER_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) /* ------------------------ FOR ENCODER ------------------------ */ -static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int64_t size) { +static FORCE_INLINE void tInitEncoder(SEncoder* pEncoder, td_endian_t endian, uint8_t* data, int32_t size) { pEncoder->endian = endian; pEncoder->data = data; pEncoder->size = (data) ? size : 0; @@ -266,7 +266,7 @@ static FORCE_INLINE int tEncodeCStr(SEncoder* pEncoder, const char* val) { } /* ------------------------ FOR DECODER ------------------------ */ -static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int64_t size) { +static FORCE_INLINE void tInitDecoder(SDecoder* pDecoder, td_endian_t endian, uint8_t* data, int32_t size) { ASSERT(!TD_IS_NULL(data)); pDecoder->endian = endian; pDecoder->data = data; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ba3e122db3..ae36ac7216 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -172,10 +172,15 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0347) #define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0348) -// mnode-mnode +// mnode-node #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) -#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0352) +#define TSDB_CODE_MND_QNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0352) +#define TSDB_CODE_MND_QNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0353) +#define TSDB_CODE_MND_SNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0354) +#define TSDB_CODE_MND_SNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0355) +#define TSDB_CODE_MND_BNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0356) +#define TSDB_CODE_MND_BNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0357) // mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/include/util/tfile.h b/include/util/tfile.h index af4c19e7d1..b3d141c443 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -38,6 +38,7 @@ int64_t tfOpenCreateWriteAppend(const char *pathname); int64_t tfClose(int64_t tfd); int64_t tfWrite(int64_t tfd, void *buf, int64_t count); int64_t tfRead(int64_t tfd, void *buf, int64_t count); +int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); int32_t tfFsync(int64_t tfd); bool tfValid(int64_t tfd); int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); @@ -47,4 +48,4 @@ int32_t tfFtruncate(int64_t tfd, int64_t length); } #endif -#endif /*_TD_UTIL_FILE_H*/ +#endif /*_TD_UTIL_FILE_H*/ diff --git a/include/util/tlog.h b/include/util/tlog.h index 5c91398cdc..a367243a46 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -37,7 +37,6 @@ extern int32_t mqttDebugFlag; extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; -extern int32_t odbcDebugFlag; extern int32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t sDebugFlag; diff --git a/include/util/tqueue.h b/include/util/tqueue.h index bcb9aea856..a57bdb5ce8 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -22,59 +22,57 @@ extern "C" { /* -This set of API for queue is designed specially for vnode/mnode. The main purpose is to -consume all the items instead of one item from a queue by one single read. Also, it can -combine multiple queues into a queue set, a consumer thread can consume a queue set via +This set of API for queue is designed specially for vnode/mnode. The main purpose is to +consume all the items instead of one item from a queue by one single read. Also, it can +combine multiple queues into a queue set, a consumer thread can consume a queue set via a single API instead of looping every queue by itself. Notes: -1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe +1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe 2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe. 3: read/write operation APIs are multi-thread safe To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c) -shall be used to set up the protection. +shall be used to set up the protection. */ -typedef void *taos_queue; -typedef void *taos_qset; -typedef void *taos_qall; +typedef struct STaosQueue STaosQueue; +typedef struct STaosQset STaosQset; +typedef struct STaosQall STaosQall; typedef void (*FProcessItem)(void *ahandle, void *pItem); -typedef void (*FProcessItems)(void *ahandle, taos_qall qall, int numOfItems); +typedef void (*FProcessItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); -taos_queue taosOpenQueue(); -void taosCloseQueue(taos_queue); -void taosSetQueueFp(taos_queue, FProcessItem, FProcessItems); -void *taosAllocateQitem(int size); -void taosFreeQitem(void *pItem); -int taosWriteQitem(taos_queue, void *pItem); -int taosReadQitem(taos_queue, void **pItem); -bool taosQueueEmpty(taos_queue); +STaosQueue *taosOpenQueue(); +void taosCloseQueue(STaosQueue *queue); +void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp); +void *taosAllocateQitem(int32_t size); +void taosFreeQitem(void *pItem); +int32_t taosWriteQitem(STaosQueue *queue, void *pItem); +int32_t taosReadQitem(STaosQueue *queue, void **ppItem); +bool taosQueueEmpty(STaosQueue *queue); -taos_qall taosAllocateQall(); -void taosFreeQall(taos_qall); -int taosReadAllQitems(taos_queue, taos_qall); -int taosGetQitem(taos_qall, void **pItem); -void taosResetQitems(taos_qall); +STaosQall *taosAllocateQall(); +void taosFreeQall(STaosQall *qall); +int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall); +int32_t taosGetQitem(STaosQall *qall, void **ppItem); +void taosResetQitems(STaosQall *qall); -taos_qset taosOpenQset(); -void taosCloseQset(); -void taosQsetThreadResume(taos_qset param); -int taosAddIntoQset(taos_qset, taos_queue, void *ahandle); -void taosRemoveFromQset(taos_qset, taos_queue); -int taosGetQueueNumber(taos_qset); +STaosQset *taosOpenQset(); +void taosCloseQset(STaosQset *qset); +void taosQsetThreadResume(STaosQset *qset); +int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); +void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); +int32_t taosGetQueueNumber(STaosQset *qset); -int taosReadQitemFromQset(taos_qset, void **pItem, void **ahandle, FProcessItem *); -int taosReadAllQitemsFromQset(taos_qset, taos_qall, void **ahandle, FProcessItems *); +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp); +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp); -int taosGetQueueItemsNumber(taos_queue param); -int taosGetQsetItemsNumber(taos_qset param); +int32_t taosGetQueueItemsNumber(STaosQueue *queue); +int32_t taosGetQsetItemsNumber(STaosQset *qset); #ifdef __cplusplus } #endif #endif /*_TD_UTIL_QUEUE_H*/ - - diff --git a/include/util/tworker.h b/include/util/tworker.h index 2e5852cbba..27f03bd2b6 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -35,7 +35,7 @@ typedef struct SWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - taos_qset qset; + STaosQset *qset; const char *name; SWorker *workers; pthread_mutex_t mutex; @@ -44,8 +44,8 @@ typedef struct SWorkerPool { typedef struct SMWorker { int32_t id; // worker id pthread_t thread; // thread - taos_qall qall; - taos_qset qset; // queue set + STaosQall *qall; + STaosQset *qset; // queue set SMWorkerPool *pool; } SMWorker; @@ -57,15 +57,15 @@ typedef struct SMWorkerPool { pthread_mutex_t mutex; } SMWorkerPool; -int32_t tWorkerInit(SWorkerPool *pool); -void tWorkerCleanup(SWorkerPool *pool); -taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp); -void tWorkerFreeQueue(SWorkerPool *pool, taos_queue queue); +int32_t tWorkerInit(SWorkerPool *pool); +void tWorkerCleanup(SWorkerPool *pool); +STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp); +void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue); -int32_t tMWorkerInit(SMWorkerPool *pool); -void tMWorkerCleanup(SMWorkerPool *pool); -taos_queue tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp); -void tMWorkerFreeQueue(SMWorkerPool *pool, taos_queue queue); +int32_t tMWorkerInit(SMWorkerPool *pool); +void tMWorkerCleanup(SMWorkerPool *pool); +STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp); +void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue); #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 021f650254..7aa4bc9ee9 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,10 +101,17 @@ typedef struct SReqResultInfo { uint32_t current; } SReqResultInfo; +typedef struct SShowReqInfo { + int64_t execId; // showId/queryId + int32_t vgId; + SArray *pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. +} SShowReqInfo; + typedef struct SRequestSendRecvBody { tsem_t rspSem; // not used now void* fp; - int64_t execId; // showId/queryId + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; SReqResultInfo resInfo; } SRequestSendRecvBody; @@ -121,6 +128,7 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; + uint64_t affectedRows; SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -131,7 +139,7 @@ extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); -SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj*); +SMsgSendInfo* buildMsgInfoImpl(SRequestObj*); int taos_init(); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index db1ea435f1..960ba95324 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -18,6 +18,7 @@ #include "clientInt.h" #include "clientLog.h" #include "query.h" +#include "scheduler.h" #include "tmsg.h" #include "tcache.h" #include "tconfig.h" @@ -230,6 +231,8 @@ void taos_init_imp(void) { SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100}; catalogInit(&cfg); + SSchedulerCfg scfg = {.maxJobNum = 100}; + schedulerInit(&scfg); tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); taosSetCoreDump(true); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6ecd9a59f7..050e763919 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -156,19 +156,12 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { }; cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - // todo OPT performance - char buf[12] = {0}; - sprintf(buf, "%"PRId64, pTscObj->pAppInfo->clusterId); - - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(buf, &pCatalog); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.ctx.pCatalog); if (code != TSDB_CODE_SUCCESS) { tfree(cxt.ctx.db); return code; } - cxt.ctx.pCatalog = pCatalog; code = qParseQuerySql(&cxt, pQuery); tfree(cxt.ctx.db); @@ -181,10 +174,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { pRequest->body.requestMsg = (SDataBuf){.pData = pDcl->pMsg, .len = pDcl->msgLen}; STscObj* pTscObj = pRequest->pTscObj; - SMsgSendInfo* pSendMsg = buildSendMsgInfoImpl(pRequest); + SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; - if (pDcl->msgType == TDMT_VND_CREATE_TABLE) { + if (pDcl->msgType == TDMT_VND_CREATE_TABLE || pDcl->msgType == TDMT_VND_SHOW_TABLES) { + if (pDcl->msgType == TDMT_VND_SHOW_TABLES) { + SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; + if (pShowReqInfo->pArray == NULL) { + pShowReqInfo->currentIndex = 0; + pShowReqInfo->pArray = pDcl->pExtension; + } + } asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg); } else { SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; @@ -196,7 +196,15 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { return TSDB_CODE_SUCCESS; } +int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQuery, SQueryDag** pDag) { + pRequest->type = pQuery->type; + return qCreateQueryDag(pQuery, pDag); +} + int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { + if (TSDB_SQL_INSERT == pRequest->type) { + return scheduleExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob, &pRequest->affectedRows); + } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); } @@ -210,6 +218,9 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + +//temporary disabled until planner ready +#if 0 CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return); //TODO: check sql valid @@ -219,19 +230,28 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq if(dagStr == NULL) { //TODO } +#endif SCMCreateTopicReq req = { .name = (char*)name, .igExists = 0, - .phyPlan = dagStr, + /*.physicalPlan = dagStr,*/ + .physicalPlan = (char*)sql, + .logicalPlan = "", }; - void* buf = NULL; - int tlen = tSerializeSCMCreateTopicReq(&buf, &req); + int tlen = tSerializeSCMCreateTopicReq(NULL, &req); + void* buf = malloc(tlen); + if(buf == NULL) { + goto _return; + } + void* abuf = buf; + tSerializeSCMCreateTopicReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet; int64_t transporterId = 0; @@ -239,8 +259,6 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq tsem_wait(&pRequest->body.rspSem); - destroySendMsgInfo(body); - _return: qDestroyQuery(pQuery); qDestroyQueryDag(pDag); @@ -273,7 +291,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQuery)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); } else { - CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return); + CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); } @@ -480,9 +498,38 @@ void* doFetchRow(SRequestObj* pRequest) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { - pRequest->type = TDMT_MND_SHOW_RETRIEVE; + if (pRequest->type == TDMT_MND_SHOW) { + pRequest->type = TDMT_MND_SHOW_RETRIEVE; + } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { + pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { + pRequest->type = TDMT_VND_SHOW_TABLES; + SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; + pShowReqInfo->currentIndex += 1; + if (pShowReqInfo->currentIndex >= taosArrayGetSize(pShowReqInfo->pArray)) { + return NULL; + } - SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest); + SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); + SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); + pShowReq->head.vgId = htonl(pVgroupInfo->vgId); + + pRequest->body.requestMsg.len = sizeof(SVShowTablesReq); + pRequest->body.requestMsg.pData = pShowReq; + + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + + int64_t transporterId = 0; + STscObj *pTscObj = pRequest->pTscObj; + asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); + + tsem_wait(&pRequest->body.rspSem); + destroySendMsgInfo(body); + + pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; + } + + SMsgSendInfo* body = buildMsgInfoImpl(pRequest); int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f7cf661019..e402403496 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -18,23 +18,26 @@ #include "tname.h" #include "clientInt.h" #include "clientLog.h" -#include "trpc.h" int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); +static void setErrno(SRequestObj* pRequest, int32_t code) { + pRequest->code = code; + terrno = code; +} + int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; - pRequest->code = code; + setErrno(pRequest, code); + sem_post(&pRequest->body.rspSem); - return 0; + return code; } int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; - terrno = code; - + setErrno(pRequest, code); sem_post(&pRequest->body.rspSem); return code; } @@ -74,52 +77,55 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -static int32_t buildRetrieveMnodeMsg(SRequestObj *pRequest, SMsgSendInfo* pMsgSendInfo) { - pMsgSendInfo->msgType = TDMT_MND_SHOW_RETRIEVE; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); - pMsgSendInfo->requestObjRefId = pRequest->self; - pMsgSendInfo->param = pRequest; - pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; - - SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); - if (pRetrieveMsg == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pRetrieveMsg->showId = htonl(pRequest->body.execId); - pMsgSendInfo->msgInfo.pData = pRetrieveMsg; - return TSDB_CODE_SUCCESS; -} - -SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) { +SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { - buildRetrieveMnodeMsg(pRequest, pMsgSendInfo); + pMsgSendInfo->requestObjRefId = pRequest->self; + pMsgSendInfo->requestId = pRequest->requestId; + pMsgSendInfo->param = pRequest; + pMsgSendInfo->msgType = pRequest->type; + + if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { + if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { + SRetrieveTableMsg* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); + if (pRetrieveMsg == NULL) { + return NULL; + } + + pRetrieveMsg->showId = htobe64(pRequest->body.showInfo.execId); + pMsgSendInfo->msgInfo.pData = pRetrieveMsg; + pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg); + } else { + SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); + if (pFetchMsg == NULL) { + return NULL; + } + + pFetchMsg->id = htobe64(pRequest->body.showInfo.execId); + pFetchMsg->head.vgId = htonl(pRequest->body.showInfo.vgId); + + pMsgSendInfo->msgInfo.pData = pFetchMsg; + pMsgSendInfo->msgInfo.len = sizeof(SVShowTablesFetchReq); + } } else { assert(pRequest != NULL); - pMsgSendInfo->requestObjRefId = pRequest->self; pMsgSendInfo->msgInfo = pRequest->body.requestMsg; - pMsgSendInfo->msgType = pRequest->type; - pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->param = pRequest; - - pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; } + pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; return pMsgSendInfo; } int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; } SShowRsp* pShow = (SShowRsp *)pMsg->pData; - pShow->showId = htonl(pShow->showId); + pShow->showId = htobe64(pShow->showId); STableMetaMsg *pMetaMsg = &(pShow->tableMeta); pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); @@ -128,6 +134,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pMetaMsg->tuid = htobe64(pMetaMsg->tuid); for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { pSchema->bytes = htonl(pSchema->bytes); + pSchema->colId = htonl(pSchema->colId); pSchema++; } @@ -148,34 +155,81 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); - pRequest->body.execId = pShow->showId; + pRequest->body.showInfo.execId = pShow->showId; + + // todo + if (pRequest->type == TDMT_VND_SHOW_TABLES) { + SShowReqInfo* pShowInfo = &pRequest->body.showInfo; + + int32_t index = pShowInfo->currentIndex; + SVgroupInfo* pInfo = taosArrayGet(pShowInfo->pArray, index); + pShowInfo->vgId = pInfo->vgId; + } + tsem_post(&pRequest->body.rspSem); return 0; } int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) { - assert(pMsg->len >= sizeof(SRetrieveTableRsp)); + SRequestObj *pRequest = param; + SReqResultInfo *pResInfo = &pRequest->body.resInfo; + tfree(pResInfo->pRspMsg); - SRequestObj* pRequest = param; -// tfree(pRequest->body.resInfo.pRspMsg); -// pRequest->body.resInfo.pRspMsg = pMsg->pData; + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + + assert(pMsg->len >= sizeof(SRetrieveTableRsp)); SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg->pData; pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); pRetrieve->precision = htons(pRetrieve->precision); - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - - tfree(pResInfo->pRspMsg); pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; - pResInfo->pData = pRetrieve->data; // todo fix this in async model + pResInfo->pData = pRetrieve->data; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, - pRetrieve->completed, pRequest->body.execId); + pRetrieve->completed, pRequest->body.showInfo.execId); + + tsem_post(&pRequest->body.rspSem); + return 0; +} + +int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) { + SRequestObj* pRequest = param; + + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + tfree(pResInfo->pRspMsg); + + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + + assert(pMsg->len >= sizeof(SRetrieveTableRsp)); + + pResInfo->pRspMsg = pMsg->pData; + + SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *) pMsg->pData; + pFetchRsp->numOfRows = htonl(pFetchRsp->numOfRows); + pFetchRsp->precision = htons(pFetchRsp->precision); + + pResInfo->pRspMsg = pMsg->pData; + pResInfo->numOfRows = pFetchRsp->numOfRows; + pResInfo->pData = pFetchRsp->data; + + pResInfo->current = 0; + setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); + + tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pFetchRsp->numOfRows, + pFetchRsp->completed, pRequest->body.showInfo.execId); tsem_post(&pRequest->body.rspSem); return 0; @@ -191,34 +245,48 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { - pRequest->code = code; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); return code; } - SUseDbRsp* pUseDbRsp = (SUseDbRsp*)pMsg->pData; - SName name = {0}; - tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT | T_NAME_DB); + SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg->pData; + SName name = {0}; + tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); char db[TSDB_DB_NAME_LEN] = {0}; tNameGetDbName(&name, db); setConnectionDB(pRequest->pTscObj, db); - tsem_post(&pRequest->body.rspSem); return 0; } int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { - assert(pMsg != NULL); + assert(pMsg != NULL && param != NULL); SRequestObj* pRequest = param; + + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + tsem_post(&pRequest->body.rspSem); + return code; } int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { // todo: Remove cache in catalog cache. SRequestObj* pRequest = param; + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + tsem_post(&pRequest->body.rspSem); + return code; } void initMsgHandleFp() { @@ -304,4 +372,7 @@ void initMsgHandleFp() { handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; + + handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = processShowRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = processRetrieveVndRsp; } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index be001780ca..97692d71b4 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -51,353 +51,355 @@ TEST(testCase, driverInit_Test) { taos_init(); } TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - taos_close(pConn); -} - -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + if (pConn == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); } - - taos_free_result(pRes); taos_close(pConn); } -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//TEST(testCase, create_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} - taos_close(pConn); -} +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +//TEST(testCase, drop_db_test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// showDB(pConn); +//// +//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// showDB(pConn); +//// +//// pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// taos_close(pConn); +//} - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - -TEST(testCase, drop_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - showDB(pConn); - - TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - showDB(pConn); - - pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("create to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - taos_close(pConn); -} - - TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - // assert(pConn != NULL); - // - // TAOS_RES* pRes = taos_query(pConn, "use abc1"); - // taos_free_result(pRes); - // - // pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - // taos_free_result(pRes); - // - // taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} +// TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_table_Test) { +// // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// // assert(pConn != NULL); +// // +// // TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// // taos_free_result(pRes); +// // +// // pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// // taos_free_result(pRes); +// // +// // taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +//// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +//// } +//// +//// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} //TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -441,7 +443,50 @@ TEST(testCase, show_table_Test) { taos_free_result(pRes); pRes = taos_query(pConn, "show tables"); - taos_free_result(pRes); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); taos_close(pConn); } + +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1) t_3 using st2 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 910d05e9a3..9ddadc9ba6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -15,17 +15,18 @@ #define _DEFAULT_SOURCE #include "os.h" + #include "taosdef.h" #include "taoserror.h" -#include "ulog.h" -#include "tlog.h" -#include "tconfig.h" -#include "tglobal.h" #include "tcompare.h" -#include "tutil.h" -#include "ttimezone.h" -#include "tlocale.h" +#include "tconfig.h" #include "tep.h" +#include "tglobal.h" +#include "tlocale.h" +#include "tlog.h" +#include "ttimezone.h" +#include "tutil.h" +#include "ulog.h" // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -36,22 +37,24 @@ uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second int8_t tsEnableTelemetryReporting = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; +int32_t tsNumOfSupportVnodes = 16; // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; //disable this, means query, show command use udp protocol as default -int32_t tsMaxShellConns = 50000; +int32_t tsRpcTimer = 300; +int32_t tsRpcMaxTime = 600; // seconds; +int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default +int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; -int32_t tsShellActivityTimer = 3; // second +int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; -int8_t tsDaylight = 0; +int8_t tsDaylight = 0; int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -int64_t tsMaxVnodeQueuedBytes = 1024*1024*1024; //1GB - +int8_t tsEnableSlaveQuery = 1; +int8_t tsEnableAdjustMaster = 1; +int8_t tsPrintAuth = 0; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -79,7 +82,7 @@ int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; -int8_t tsTscEnableRecordSql = 0; +int8_t tsTscEnableRecordSql = 0; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -89,7 +92,7 @@ int32_t tsMaxNumOfOrderedResults = 100000; int32_t tsMinSlidingTime = 10; // the maxinum number of distict query result -int32_t tsMaxNumOfDistinctResults = 1000 * 10000; +int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 us for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; @@ -101,7 +104,7 @@ int32_t tsMaxStreamComputDelay = 20000; int32_t tsStreamCompStartDelay = 10000; // the stream computing delay time after executing failed, change accordingly -int32_t tsRetryStreamCompDelay = 10*1000; +int32_t tsRetryStreamCompDelay = 10 * 1000; // The delayed computing ration. 10% of the whole computing time window by default. float tsStreamComputDelayRatio = 0.1f; @@ -120,30 +123,16 @@ int64_t tsQueryBufferSizeBytes = -1; int32_t tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int8_t tsKeepOriginalColumnName = 0; +int8_t tsKeepOriginalColumnName = 0; -// tsdb config +// long query death-lock +int8_t tsDeadLockKillQuery = 0; + +// tsdb config // For backward compatibility bool tsdbForceKeepFile = false; -// balance -int8_t tsEnableFlowCtrl = 1; -int8_t tsEnableSlaveQuery = 1; -int8_t tsEnableAdjustMaster = 1; - - -// monitor -char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; -char tsInternalPass[] = "secretkey"; - -// internal -int8_t tsCompactMnodeWal = 0; -int8_t tsPrintAuth = 0; -char tsVnodeDir[PATH_MAX] = {0}; -char tsDnodeDir[PATH_MAX] = {0}; -char tsMnodeDir[PATH_MAX] = {0}; - -int32_t tsDiskCfgNum = 0; +int32_t tsDiskCfgNum = 0; #ifndef _STORAGE SDiskCfg tsDiskCfg[1]; @@ -160,31 +149,28 @@ SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; // system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; -int32_t tsTotalMemoryMB = 0; +float tsTotalTmpDirGB = 0; +float tsTotalDataDirGB = 0; +float tsAvailTmpDirectorySpace = 0; +float tsAvailDataDirGB = 0; +float tsUsedDataDirGB = 0; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 2.0f; +int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; -#ifdef TD_TSZ // // lossy compress 6 // -char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty can close lossy compress. -// below option can take effect when tsLossyColumns not empty -double fPrecision = 1E-8; // float column precision -double dPrecision = 1E-16; // double column precision -uint32_t maxRange = 500; // max range -uint32_t curRange = 100; // range -char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR -#endif +char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty + // can close lossy compress. +// below option can take effect when tsLossyColumns not empty +double tsFPrecision = 1E-8; // float column precision +double tsDPrecision = 1E-16; // double column precision +uint32_t tsMaxRange = 500; // max range +uint32_t tsCurRange = 100; // range +char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR -// long query death-lock -int8_t tsDeadLockKillQuery = 0; int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; @@ -195,13 +181,12 @@ char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { - if (debugFlag != 0) { + if (debugFlag != 0) { mDebugFlag = debugFlag; dDebugFlag = debugFlag; vDebugFlag = debugFlag; jniDebugFlag = debugFlag; - odbcDebugFlag = debugFlag; - qDebugFlag = debugFlag; + qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; sDebugFlag = debugFlag; @@ -213,12 +198,12 @@ void taosSetAllDebugFlag() { } int32_t taosCfgDynamicOptions(char *msg) { - char *option, *value; - int32_t olen, vlen; - int32_t vint = 0; + char *option, *value; + int32_t olen, vlen; + int32_t vint = 0; paGetToken(msg, &option, &olen); - if (olen == 0) return -1;; + if (olen == 0) return -1; paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) @@ -231,9 +216,9 @@ int32_t taosCfgDynamicOptions(char *msg) { for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; - //if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; + // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; - + int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; @@ -262,7 +247,7 @@ int32_t taosCfgDynamicOptions(char *msg) { return 0; } if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); + taosSetAllDebugFlag(); } return 0; } @@ -323,7 +308,7 @@ static void doInitGlobalConfig(void) { srand(taosSafeRand()); SGlobalCfg cfg = {0}; - + // ip address cfg.option = "firstEp"; cfg.ptr = tsFirst; @@ -366,6 +351,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); + cfg.option = "supportVnodes"; + cfg.ptr = &tsNumOfSupportVnodes; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; + cfg.minValue = 0; + cfg.maxValue = 65536; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosAddConfigOption(cfg); + // directory cfg.option = "configDir"; cfg.ptr = configDir; @@ -442,8 +437,8 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsMaxNumOfDistinctResults; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10*10000; - cfg.maxValue = 10000*10000; + cfg.minValue = 10 * 10000; + cfg.maxValue = 10000 * 10000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); @@ -749,17 +744,6 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 10000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - // module configs - cfg.option = "flowctrl"; - cfg.ptr = &tsEnableFlowCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); cfg.option = "slaveQuery"; @@ -893,16 +877,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "odbcDebugFlag"; - cfg.ptr = &odbcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - cfg.option = "uDebugFlag"; cfg.ptr = &uDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1034,7 +1008,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - // enable kill long query + // enable kill long query cfg.option = "deadLockKillQuery"; cfg.ptr = &tsDeadLockKillQuery; cfg.valType = TAOS_CFG_VTYPE_INT8; @@ -1066,7 +1040,6 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); cfg.option = "dPrecision"; @@ -1100,23 +1073,20 @@ static void doInitGlobalConfig(void) { taosAddConfigOption(cfg); assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); #else - //assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); + // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); #endif - } -void taosInitGlobalCfg() { - pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); -} +void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } int32_t taosCheckAndPrintCfg() { - char fqdn[TSDB_FQDN_LEN]; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); } - + if (tsLocalFqdn[0] == 0) { taosGetFqdn(tsLocalFqdn); } @@ -1143,7 +1113,7 @@ int32_t taosCheckAndPrintCfg() { if (taosDirExist(tsTempDir) != 0) { return -1; } - + taosGetSystemInfo(); tsSetLocale(); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7f285046c..b81143ee62 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -25,4 +25,230 @@ #undef TD_MSG_INFO_ #define TD_MSG_DICT_ #undef TD_MSG_SEG_CODE_ -#include "tmsgdef.h" \ No newline at end of file +#include "tmsgdef.h" + +static int tmsgStartEncode(SMsgEncoder *pME); +static void tmsgEndEncode(SMsgEncoder *pME); +static int tmsgStartDecode(SMsgDecoder *pMD); +static void tmsgEndDecode(SMsgDecoder *pMD); + +/* ------------------------ ENCODE/DECODE FUNCTIONS ------------------------ */ +void tmsgInitMsgEncoder(SMsgEncoder *pME, td_endian_t endian, uint8_t *data, int64_t size) { + tInitEncoder(&(pME->coder), endian, data, size); + TD_SLIST_INIT(&(pME->eStack)); +} + +void tmsgClearMsgEncoder(SMsgEncoder *pME) { + struct SMEListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pME->eStack)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pME->eStack)); + free(pNode); + } +} + +void tmsgInitMsgDecoder(SMsgDecoder *pMD, td_endian_t endian, uint8_t *data, int64_t size) { + tInitDecoder(&pMD->coder, endian, data, size); + TD_SLIST_INIT(&(pMD->dStack)); + TD_SLIST_INIT(&(pMD->freeList)); +} + +void tmsgClearMsgDecoder(SMsgDecoder *pMD) { + { + struct SMDFreeListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pMD->freeList)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pMD->freeList)); + free(pNode); + } + } + { + struct SMDListNode *pNode; + for (;;) { + pNode = TD_SLIST_HEAD(&(pMD->dStack)); + if (TD_IS_NULL(pNode)) break; + TD_SLIST_POP(&(pMD->dStack)); + free(pNode); + } + } +} + +/* ------------------------ MESSAGE ENCODE/DECODE ------------------------ */ +int tmsgSVCreateTbReqEncode(SMsgEncoder *pCoder, SVCreateTbReq *pReq) { + tmsgStartEncode(pCoder); + // TODO + + tmsgEndEncode(pCoder); + return 0; +} + +int tmsgSVCreateTbReqDecode(SMsgDecoder *pCoder, SVCreateTbReq *pReq) { + tmsgStartDecode(pCoder); + + // TODO: decode + + // Decode is not end + if (pCoder->coder.pos != pCoder->coder.size) { + // Continue decode + } + + tmsgEndDecode(pCoder); + return 0; +} + +int tSerializeSVCreateTbReq(void **buf, const SVCreateTbReq *pReq) { + int tlen = 0; + + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU32(buf, pReq->ttl); + tlen += taosEncodeFixedU32(buf, pReq->keep); + tlen += taosEncodeFixedU8(buf, pReq->type); + + switch (pReq->type) { + case TD_SUPER_TABLE: + tlen += taosEncodeFixedU64(buf, pReq->stbCfg.suid); + tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); + for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); + } + tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols); + for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); + } + break; + case TD_CHILD_TABLE: + tlen += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); + tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); + break; + case TD_NORMAL_TABLE: + tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols); + for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); + tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].colId); + tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); + tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); + } + break; + default: + ASSERT(0); + } + + return tlen; +} + +void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { + buf = taosDecodeFixedU64(buf, &(pReq->ver)); + buf = taosDecodeString(buf, &(pReq->name)); + buf = taosDecodeFixedU32(buf, &(pReq->ttl)); + buf = taosDecodeFixedU32(buf, &(pReq->keep)); + buf = taosDecodeFixedU8(buf, &(pReq->type)); + + switch (pReq->type) { + case TD_SUPER_TABLE: + buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); + buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); + pReq->stbCfg.pSchema = (SSchema *)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); + buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].colId)); + buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); + buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); + } + buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols); + pReq->stbCfg.pTagSchema = (SSchema *)malloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); + buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); + buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); + } + break; + case TD_CHILD_TABLE: + buf = taosDecodeFixedU64(buf, &pReq->ctbCfg.suid); + buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); + break; + case TD_NORMAL_TABLE: + buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols); + pReq->ntbCfg.pSchema = (SSchema *)malloc(pReq->ntbCfg.nCols * sizeof(SSchema)); + for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) { + buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); + buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); + buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); + } + break; + default: + ASSERT(0); + } + + return buf; +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static int tmsgStartEncode(SMsgEncoder *pME) { + struct SMEListNode *pNode = (struct SMEListNode *)malloc(sizeof(*pNode)); + if (TD_IS_NULL(pNode)) return -1; + + pNode->coder = pME->coder; + TD_SLIST_PUSH(&(pME->eStack), pNode); + TD_CODER_MOVE_POS(&(pME->coder), sizeof(int32_t)); + + return 0; +} + +static void tmsgEndEncode(SMsgEncoder *pME) { + int32_t size; + struct SMEListNode *pNode; + + pNode = TD_SLIST_HEAD(&(pME->eStack)); + ASSERT(pNode); + TD_SLIST_POP(&(pME->eStack)); + + size = pME->coder.pos - pNode->coder.pos; + tEncodeI32(&(pNode->coder), size); + + free(pNode); +} + +static int tmsgStartDecode(SMsgDecoder *pMD) { + struct SMDListNode *pNode; + int32_t size; + + pNode = (struct SMDListNode *)malloc(sizeof(*pNode)); + if (pNode == NULL) return -1; + + tDecodeI32(&(pMD->coder), &size); + + pNode->coder = pMD->coder; + TD_SLIST_PUSH(&(pMD->dStack), pNode); + + pMD->coder.pos = 0; + pMD->coder.size = size - sizeof(int32_t); + pMD->coder.data = TD_CODER_CURRENT(&(pNode->coder)); + + return 0; +} + +static void tmsgEndDecode(SMsgDecoder *pMD) { + ASSERT(pMD->coder.pos == pMD->coder.size); + struct SMDListNode *pNode; + + pNode = TD_SLIST_HEAD(&(pMD->dStack)); + ASSERT(pNode); + TD_SLIST_POP(&(pMD->dStack)); + + pNode->coder.pos += pMD->coder.size; + + pMD->coder = pNode->coder; + + free(pNode); +} \ No newline at end of file diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 88a5ebb7f6..f8ef9f0979 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -259,3 +259,13 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return 0; } + +SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name) { + SSchema s = {0}; + s.type = type; + s.bytes = bytes; + s.colId = colId; + + tstrncpy(s.name, name, tListLen(s.name)); + return s; +} \ No newline at end of file diff --git a/source/common/test/CMakeLists.txt b/source/common/test/CMakeLists.txt index 03ca95ff64..58dde913f0 100644 --- a/source/common/test/CMakeLists.txt +++ b/source/common/test/CMakeLists.txt @@ -18,11 +18,11 @@ TARGET_INCLUDE_DIRECTORIES( ) # tmsg test -add_executable(tmsgTest "") -target_sources(tmsgTest - PRIVATE - "tmsgTest.cpp" - "../src/tmsg.c" -) -target_include_directories(tmsgTest PUBLIC "${CMAKE_SOURCE_DIR}/include/common/") -target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) \ No newline at end of file +# add_executable(tmsgTest "") +# target_sources(tmsgTest +# PRIVATE +# "tmsgTest.cpp" +# "../src/tmsg.c" +# ) +# target_include_directories(tmsgTest PUBLIC "${CMAKE_SOURCE_DIR}/include/common/") +# target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 70dca0e4df..cedab6266e 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -139,8 +139,8 @@ void dmnWaitSignal() { void dmnInitOption(SDnodeOpt *pOption) { pOption->sver = 30000000; //3.0.0.0 pOption->numOfCores = tsNumOfCores; - pOption->numOfSupportVnodes = 16; - pOption->numOfCommitThreads = 1; + pOption->numOfSupportVnodes = tsNumOfSupportVnodes; + pOption->numOfCommitThreads = tsNumOfCommitThreads; pOption->statusInterval = tsStatusInterval; pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; pOption->ratioOfQueryCores = tsRatioOfQueryCores; diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 954e21aefa..07c8ce5d02 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -64,7 +64,7 @@ typedef struct { int32_t maxNum; void *queueFp; SDnode *pDnode; - taos_queue queue; + STaosQueue *queue; union { SWorkerPool pool; SMWorkerPool mpool; @@ -92,7 +92,7 @@ typedef struct { SDnodeEps *dnodeEps; pthread_t *threadId; SRWLatch latch; - taos_queue pMgmtQ; + STaosQueue *pMgmtQ; SWorkerPool mgmtPool; } SDnodeMgmt; diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 992f6ac0a1..c12d449517 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -19,7 +19,7 @@ #include "dndTransport.h" #include "dndWorker.h" -static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs); static SBnode *dndAcquireBnode(SDnode *pDnode) { SBnodeMgmt *pMgmt = &pDnode->bmgmt; @@ -256,7 +256,7 @@ static int32_t dndDropBnode(SDnode *pDnode) { } int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateBnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateBnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -268,7 +268,7 @@ int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropBnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropBnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -286,7 +286,7 @@ static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) { taosFreeQitem(pMsg); } -static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t code) { +static void dndSendBnodeErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { for (int32_t i = 0; i < numOfMsgs; ++i) { SRpcMsg *pMsg = NULL; taosGetQitem(qall, (void **)&pMsg); @@ -294,7 +294,7 @@ static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t cod } } -static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessBnodeQueue(SDnode *pDnode, STaosQall *qall, int32_t numOfMsgs) { SBnode *pBnode = dndAcquireBnode(pDnode); if (pBnode == NULL) { dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 98763fce30..30b069f349 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -15,9 +15,12 @@ #define _DEFAULT_SOURCE #include "dndDnode.h" +#include "dndBnode.h" +#include "dndMnode.h" +#include "dndQnode.h" +#include "dndSnode.h" #include "dndTransport.h" #include "dndVnodes.h" -#include "dndMnode.h" static int32_t dndInitMgmtWorker(SDnode *pDnode); static void dndCleanupMgmtWorker(SDnode *pDnode); @@ -367,8 +370,8 @@ void dndSendStatusMsg(SDnode *pDnode) { pStatus->clusterId = htobe64(pMgmt->clusterId); pStatus->rebootTime = htobe64(pMgmt->rebootTime); pStatus->updateTime = htobe64(pMgmt->updateTime); - pStatus->numOfCores = htons(pDnode->opt.numOfCores); - pStatus->numOfSupportVnodes = htons(pDnode->opt.numOfSupportVnodes); + pStatus->numOfCores = htonl(pDnode->opt.numOfCores); + pStatus->numOfSupportVnodes = htonl(pDnode->opt.numOfSupportVnodes); tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN); pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval); @@ -394,7 +397,7 @@ void dndSendStatusMsg(SDnode *pDnode) { static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:% " PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosWLockLatch(&pMgmt->latch); pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; @@ -437,19 +440,21 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pMsg) { } SStatusRsp *pRsp = pMsg->pCont; - SDnodeCfg *pCfg = &pRsp->dnodeCfg; - pCfg->dnodeId = htonl(pCfg->dnodeId); - pCfg->clusterId = htobe64(pCfg->clusterId); - dndUpdateDnodeCfg(pDnode, pCfg); + if (pMsg->pCont != NULL && pMsg->contLen != 0) { + SDnodeCfg *pCfg = &pRsp->dnodeCfg; + pCfg->dnodeId = htonl(pCfg->dnodeId); + pCfg->clusterId = htobe64(pCfg->clusterId); + dndUpdateDnodeCfg(pDnode, pCfg); - SDnodeEps *pDnodeEps = &pRsp->dnodeEps; - pDnodeEps->num = htonl(pDnodeEps->num); - for (int32_t i = 0; i < pDnodeEps->num; ++i) { - pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); - pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); + SDnodeEps *pDnodeEps = &pRsp->dnodeEps; + pDnodeEps->num = htonl(pDnodeEps->num); + for (int32_t i = 0; i < pDnodeEps->num; ++i) { + pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); + pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); + } + + dndUpdateDnodeEps(pDnode, pDnodeEps); } - - dndUpdateDnodeEps(pDnode, pDnodeEps); pMgmt->statusSent = 0; } @@ -648,6 +653,24 @@ static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { case TDMT_DND_DROP_MNODE: code = dndProcessDropMnodeReq(pDnode, pMsg); break; + case TDMT_DND_CREATE_QNODE: + code = dndProcessCreateQnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_QNODE: + code = dndProcessDropQnodeReq(pDnode, pMsg); + break; + case TDMT_DND_CREATE_SNODE: + code = dndProcessCreateSnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_SNODE: + code = dndProcessDropSnodeReq(pDnode, pMsg); + break; + case TDMT_DND_CREATE_BNODE: + code = dndProcessCreateBnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_BNODE: + code = dndProcessDropBnodeReq(pDnode, pMsg); + break; case TDMT_DND_CONFIG_DNODE: code = dndProcessConfigDnodeReq(pDnode, pMsg); break; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 8fb95c0b75..577cb0c3b0 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -299,7 +299,7 @@ static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); } -static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SCreateMnodeInMsg *pMsg) { +static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SDCreateMnodeMsg *pMsg) { dndInitMnodeOption(pDnode, pOption); pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); @@ -417,8 +417,8 @@ static int32_t dndDropMnode(SDnode *pDnode) { return 0; } -static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { - SCreateMnodeInMsg *pMsg = pRpcMsg->pCont; +static SDCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { + SDCreateMnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); for (int32_t i = 0; i < pMsg->replica; ++i) { pMsg->replicas[i].id = htonl(pMsg->replicas[i].id); @@ -429,7 +429,7 @@ static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { } int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); + SDCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -445,7 +445,7 @@ int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SAlterMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); + SDAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -465,7 +465,7 @@ int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropMnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropMnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 5d04a4f449..1f3d6ee371 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -261,7 +261,7 @@ static int32_t dndDropQnode(SDnode *pDnode) { } int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateQnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateQnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -273,7 +273,7 @@ int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropQnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropQnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 151fc7e6a1..ab4ca191a9 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -256,7 +256,7 @@ static int32_t dndDropSnode(SDnode *pDnode) { } int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateSnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateSnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -268,7 +268,7 @@ int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropSnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropSnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8db09ea4cd..e5be16937b 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -37,6 +37,18 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg; @@ -66,6 +78,12 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; @@ -85,7 +103,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_HEARTBEAT)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STATUS_RSP)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TRANS)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TRANS_RSP)] = dndProcessMnodeWriteMsg; @@ -123,6 +141,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CREATE_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; } static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -365,6 +385,7 @@ void dndCleanupTrans(SDnode *pDnode) { void dndSendMsgToDnode(SDnode *pDnode, SEpSet *pEpSet, SRpcMsg *pMsg) { STransMgmt *pMgmt = &pDnode->tmgmt; + if (pMgmt->clientRpc == NULL) return; rpcSendRequest(pMgmt->clientRpc, pEpSet, pMsg, NULL); } diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 3eab3e5aec..8835e0ba65 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -27,20 +27,20 @@ typedef struct { } SWrapperCfg; typedef struct { - int32_t vgId; - int32_t refCount; - int32_t vgVersion; - int8_t dropped; - int8_t accessState; - uint64_t dbUid; - char *db; - char *path; - SVnode *pImpl; - taos_queue pWriteQ; - taos_queue pSyncQ; - taos_queue pApplyQ; - taos_queue pQueryQ; - taos_queue pFetchQ; + int32_t vgId; + int32_t refCount; + int32_t vgVersion; + int8_t dropped; + int8_t accessState; + uint64_t dbUid; + char *db; + char *path; + SVnode *pImpl; + STaosQueue *pWriteQ; + STaosQueue *pSyncQ; + STaosQueue *pApplyQ; + STaosQueue *pQueryQ; + STaosQueue* pFetchQ; } SVnodeObj; typedef struct { @@ -72,9 +72,9 @@ static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); +static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); +static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); +static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs); void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); @@ -768,7 +768,7 @@ static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp); } -static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -804,7 +804,7 @@ static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t taosArrayDestroy(pArray); } -static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -815,7 +815,7 @@ static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t } } -static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { +static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { @@ -826,7 +826,7 @@ static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t } } -static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { +static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) { int32_t code = 0; if (pQueue == NULL) { diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index dcce270d7d..b0596bed08 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -8,6 +8,9 @@ add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) +add_subdirectory(qnode) +add_subdirectory(snode) +add_subdirectory(bnode) add_subdirectory(profile) add_subdirectory(show) add_subdirectory(stb) diff --git a/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt new file mode 100644 index 0000000000..ea769a701c --- /dev/null +++ b/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. STEST_SRC) +add_executable(dnode_test_bnode ${STEST_SRC}) +target_link_libraries( + dnode_test_bnode + PUBLIC sut +) + +add_test( + NAME dnode_test_bnode + COMMAND dnode_test_bnode +) diff --git a/source/dnode/mgmt/impl/test/bnode/bnode.cpp b/source/dnode/mgmt/impl/test/bnode/bnode.cpp new file mode 100644 index 0000000000..b1a85522c4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/bnode/bnode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestBnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_bnode1", 9068); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9068"; + + server2.Start("/tmp/dnode_test_bnode2", fqdn, 9069, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestBnode::test; +TestServer DndTestBnode::server2; + +TEST_F(DndTestBnode, 01_ShowBnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + CHECK_META("show bnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestBnode, 02_Create_Bnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + CHECK_META("show bnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestBnode, 03_Create_Bnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestBnode, 04_Create_Bnode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9069); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create bnode + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckBinary("localhost:9069", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop bnode + int32_t contLen = sizeof(SMDropBnodeMsg); + + SMDropBnodeMsg* pReq = (SMDropBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index ec2c2d9a44..54d7e73be6 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -57,7 +57,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); - CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "support_vnodes"); CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); @@ -68,7 +68,7 @@ TEST_F(DndTestDnode, 01_ShowDnode) { CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); - CheckInt16(1); + CheckInt16(16); CheckBinary("ready", 10); CheckTimestamp(); CheckBinary("", 24); @@ -112,8 +112,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("localhost:9042", TSDB_EP_LEN); CheckInt16(0); CheckInt16(0); - CheckInt16(1); - CheckInt16(1); + CheckInt16(16); + CheckInt16(16); CheckBinary("ready", 10); CheckBinary("ready", 10); CheckTimestamp(); @@ -140,7 +140,7 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); - CheckInt16(1); + CheckInt16(16); CheckBinary("ready", 10); CheckTimestamp(); CheckBinary("", 24); @@ -199,10 +199,10 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(0); CheckInt16(0); CheckInt16(0); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); CheckBinary("ready", 10); CheckBinary("ready", 10); CheckBinary("ready", 10); @@ -242,10 +242,10 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckInt16(0); CheckInt16(0); CheckInt16(0); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); - CheckInt16(1); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); + CheckInt16(16); CheckBinary("ready", 10); CheckBinary("ready", 10); CheckBinary("ready", 10); diff --git a/source/dnode/mgmt/impl/test/mnode/mnode.cpp b/source/dnode/mgmt/impl/test/mnode/mnode.cpp index a6cec93539..e9b1ef45bd 100644 --- a/source/dnode/mgmt/impl/test/mnode/mnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/mnode.cpp @@ -72,9 +72,9 @@ TEST_F(DndTestMnode, 01_ShowDnode) { TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SCreateMnodeMsg); + int32_t contLen = sizeof(SMCreateMnodeMsg); - SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); + SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -85,9 +85,9 @@ TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) { TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) { { - int32_t contLen = sizeof(SCreateMnodeMsg); + int32_t contLen = sizeof(SMCreateMnodeMsg); - SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); + SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -117,9 +117,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { { // create mnode - int32_t contLen = sizeof(SCreateMnodeMsg); + int32_t contLen = sizeof(SMCreateMnodeMsg); - SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen); + SMCreateMnodeMsg* pReq = (SMCreateMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -144,9 +144,9 @@ TEST_F(DndTestMnode, 04_Create_Mnode) { { // drop mnode - int32_t contLen = sizeof(SDropMnodeMsg); + int32_t contLen = sizeof(SMDropMnodeMsg); - SDropMnodeMsg* pReq = (SDropMnodeMsg*)rpcMallocCont(contLen); + SMDropMnodeMsg* pReq = (SMDropMnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_MNODE, pReq, contLen); diff --git a/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt new file mode 100644 index 0000000000..2edb6de8b8 --- /dev/null +++ b/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. QTEST_SRC) +add_executable(dnode_test_qnode ${QTEST_SRC}) +target_link_libraries( + dnode_test_qnode + PUBLIC sut +) + +add_test( + NAME dnode_test_qnode + COMMAND dnode_test_qnode +) diff --git a/source/dnode/mgmt/impl/test/qnode/qnode.cpp b/source/dnode/mgmt/impl/test/qnode/qnode.cpp new file mode 100644 index 0000000000..40464de816 --- /dev/null +++ b/source/dnode/mgmt/impl/test/qnode/qnode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestQnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_qnode1", 9064); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9064"; + + server2.Start("/tmp/dnode_test_qnode2", fqdn, 9065, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestQnode::test; +TestServer DndTestQnode::server2; + +TEST_F(DndTestQnode, 01_ShowQnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestQnode, 02_Create_Qnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestQnode, 03_Create_Qnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestQnode, 04_Create_Qnode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9065); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create qnode + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckBinary("localhost:9065", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop qnode + int32_t contLen = sizeof(SMDropQnodeMsg); + + SMDropQnodeMsg* pReq = (SMDropQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/snode/CMakeLists.txt b/source/dnode/mgmt/impl/test/snode/CMakeLists.txt new file mode 100644 index 0000000000..180b1ec163 --- /dev/null +++ b/source/dnode/mgmt/impl/test/snode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. STEST_SRC) +add_executable(dnode_test_snode ${STEST_SRC}) +target_link_libraries( + dnode_test_snode + PUBLIC sut +) + +add_test( + NAME dnode_test_snode + COMMAND dnode_test_snode +) diff --git a/source/dnode/mgmt/impl/test/snode/snode.cpp b/source/dnode/mgmt/impl/test/snode/snode.cpp new file mode 100644 index 0000000000..a14a575beb --- /dev/null +++ b/source/dnode/mgmt/impl/test/snode/snode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestSnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_snode1", 9066); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9066"; + + server2.Start("/tmp/dnode_test_snode2", fqdn, 9067, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestSnode::test; +TestServer DndTestSnode::server2; + +TEST_F(DndTestSnode, 01_ShowSnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + CHECK_META("show snodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestSnode, 02_Create_Snode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + CHECK_META("show snodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestSnode, 03_Create_Snode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestSnode, 04_Create_Snode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9067); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create snode + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckBinary("localhost:9067", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop snode + int32_t contLen = sizeof(SMDropSnodeMsg); + + SMDropSnodeMsg* pReq = (SMDropSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index ca168fb6d7..dca0f48516 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -146,8 +146,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pSchema->bytes = htonl(pSchema->bytes); } - EXPECT_STREQ(pRsp->tbFname, ""); - EXPECT_STREQ(pRsp->stbFname, "1.d1.stb"); + EXPECT_STREQ(pRsp->tbFname, "1.d1.stb"); + EXPECT_STREQ(pRsp->stbFname, ""); EXPECT_EQ(pRsp->numOfColumns, 2); EXPECT_EQ(pRsp->numOfTags, 3); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); diff --git a/source/dnode/mgmt/impl/test/sut/src/base.cpp b/source/dnode/mgmt/impl/test/sut/src/base.cpp index 98371e9893..e14dc94d31 100644 --- a/source/dnode/mgmt/impl/test/sut/src/base.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/base.cpp @@ -24,7 +24,6 @@ void Testbase::InitLog(const char* path) { tmrDebugFlag = 0; uDebugFlag = 143; rpcDebugFlag = 0; - odbcDebugFlag = 0; qDebugFlag = 0; wDebugFlag = 0; sDebugFlag = 0; diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index f29b2fad1d..8ac5f62144 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -26,7 +26,7 @@ SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t p SDnodeOpt option = {0}; option.sver = 1; option.numOfCores = 1; - option.numOfSupportVnodes = 1; + option.numOfSupportVnodes = 16; option.numOfCommitThreads = 1; option.statusInterval = 1; option.numOfThreadsPerCore = 1; diff --git a/source/dnode/mnode/impl/inc/mndBalance.h b/source/dnode/mnode/impl/inc/mndBnode.h similarity index 81% rename from source/dnode/mnode/impl/inc/mndBalance.h rename to source/dnode/mnode/impl/inc/mndBnode.h index fa3679d033..cefc0b1859 100644 --- a/source/dnode/mnode/impl/inc/mndBalance.h +++ b/source/dnode/mnode/impl/inc/mndBnode.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_BALANCE_H_ -#define _TD_MND_BALANCE_H_ +#ifndef _TD_MND_BNODE_H_ +#define _TD_MND_BNODE_H_ #include "mndInt.h" @@ -22,11 +22,11 @@ extern "C" { #endif -int32_t mndInitBalance(SMnode *pMnode); -void mndCleanupBalance(SMnode *pMnode); +int32_t mndInitBnode(SMnode *pMnode); +void mndCleanupBnode(SMnode *pMnode); #ifdef __cplusplus } #endif -#endif /*_TD_MND_BALANCE_H_*/ +#endif /*_TD_MND_BNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h new file mode 100644 index 0000000000..60f186d7d2 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_CONSUMER_H_ +#define _TD_MND_CONSUMER_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitConsumer(SMnode *pMnode); +void mndCleanupConsumer(SMnode *pMnode); + +SConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); +void mndReleaseConsumer(SMnode *pMnode, SConsumerObj *pConsumer); + +SCGroupObj *mndAcquireCGroup(SMnode *pMnode, char *consumerGroup); +void mndReleaseCGroup(SMnode *pMnode, SCGroupObj *pCGroup); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_CONSUMER_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1a1306c3da..e26c0c5ae7 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -74,13 +74,6 @@ typedef enum { typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; -typedef enum { - DND_STATUS_OFFLINE = 0, - DND_STATUS_READY = 1, - DND_STATUS_CREATING = 2, - DND_STATUS_DROPPING = 3 -} EDndStatus; - typedef enum { DND_REASON_ONLINE = 0, DND_REASON_STATUS_MSG_TIMEOUT, @@ -124,10 +117,9 @@ typedef struct { int64_t rebootTime; int64_t lastAccessTime; int32_t accessTimes; - int16_t numOfVnodes; - int16_t numOfSupportVnodes; - int16_t numOfCores; - EDndStatus status; + int32_t numOfVnodes; + int32_t numOfSupportVnodes; + int32_t numOfCores; EDndReason offlineReason; uint16_t port; char fqdn[TSDB_FQDN_LEN]; @@ -144,6 +136,27 @@ typedef struct { SDnodeObj *pDnode; } SMnodeObj; +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SQnodeObj; + +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SSnodeObj; + +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SBnodeObj; + typedef struct { int32_t maxUsers; int32_t maxDbs; @@ -272,7 +285,7 @@ typedef struct { } SFuncObj; typedef struct { - int32_t id; + int64_t id; int8_t type; int8_t replica; int16_t numOfColumns; @@ -301,8 +314,34 @@ typedef struct { void* executor; int32_t sqlLen; char* sql; + char* logicalPlan; + char* physicalPlan; } STopicObj; +typedef struct { + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + uint64_t uid; + //uint64_t dbUid; + int32_t version; + SRWLatch lock; + +} SConsumerObj; + +typedef struct { + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + uint64_t uid; + //uint64_t dbUid; + int32_t version; + SRWLatch lock; + +} SCGroupObj; + typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; char db[TSDB_DB_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 764dfbffc1..c76186c0a2 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,7 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); -bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode); +bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index daa87af1f5..15ff65a8fc 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -41,7 +41,7 @@ typedef struct { } SMnodeStep; typedef struct { - int32_t showId; + int64_t showId; ShowMetaFp metaFps[TSDB_MGMT_TABLE_MAX]; ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; diff --git a/source/dnode/mnode/impl/src/mndBalance.c b/source/dnode/mnode/impl/inc/mndQnode.h similarity index 74% rename from source/dnode/mnode/impl/src/mndBalance.c rename to source/dnode/mnode/impl/inc/mndQnode.h index 6ce89b96a3..5d177b3f6d 100644 --- a/source/dnode/mnode/impl/src/mndBalance.c +++ b/source/dnode/mnode/impl/inc/mndQnode.h @@ -13,9 +13,20 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#include "os.h" +#ifndef _TD_MND_QNODE_H_ +#define _TD_MND_QNODE_H_ + #include "mndInt.h" -int32_t mndInitBalance(SMnode *pMnode) { return 0; } -void mndCleanupBalance(SMnode *pMnode) {} \ No newline at end of file +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitQnode(SMnode *pMnode); +void mndCleanupQnode(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_QNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndSnode.h b/source/dnode/mnode/impl/inc/mndSnode.h new file mode 100644 index 0000000000..8d64879605 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndSnode.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_SNODE_H_ +#define _TD_MND_SNODE_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitSnode(SMnode *pMnode); +void mndCleanupSnode(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_SNODE_H_*/ diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c new file mode 100644 index 0000000000..2d14e0f92e --- /dev/null +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndBnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_BNODE_VER_NUMBER 1 +#define TSDB_BNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj); +static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); +static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode); +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); + +int32_t mndInitBnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_BNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, + .insertFp = (SdbInsertFp)mndBnodeActionInsert, + .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndBnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndProcessCreateBnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndProcessDropBnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndGetBnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupBnode(SMnode *pMnode) {} + +static SBnodeObj *mndAcquireBnode(SMnode *pMnode, int32_t snodeId) { + SSdb *pSdb = pMnode->pSdb; + SBnodeObj *pObj = sdbAcquire(pSdb, SDB_BNODE, &snodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseBnode(SMnode *pMnode, SBnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_BNODE, TSDB_BNODE_VER_NUMBER, sizeof(SBnodeObj) + TSDB_BNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_BNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_BNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode snode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SBnodeObj)); + SBnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_BNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj) { + mTrace("snode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("snode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { + mTrace("snode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode) { + mTrace("snode:%d, perform update action", pOldBnode->id); + pOldBnode->updateTime = pNewBnode->updateTime; + return 0; +} + +static int32_t mndSetCreateBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDCreateBnodeMsg *pMsg = malloc(sizeof(SDCreateBnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateBnodeMsg); + action.msgType = TDMT_DND_CREATE_BNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateBnodeMsg *pCreate) { + SBnodeObj snodeObj = {0}; + snodeObj.id = pDnode->id; + snodeObj.createdTime = taosGetTimestampMs(); + snodeObj.updateTime = snodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_BNODE_OVER; + } + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateBnodeRedoLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndSetCreateBnodeCommitLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + code = 0; + +CREATE_BNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateBnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("snode:%d, start to create", pCreate->dnodeId); + + SBnodeObj *pObj = mndAcquireBnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("snode:%d, snode already exist", pObj->id); + mndReleaseBnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("snode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateBnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDDropBnodeMsg *pMsg = malloc(sizeof(SDDropBnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropBnodeMsg); + action.msgType = TDMT_DND_DROP_BNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pMsg, SBnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_BNODE_OVER; + } + + mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + + if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + code = 0; + +DROP_BNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMDropBnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("snode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SBnodeObj *pObj = mndAcquireBnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("snode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropBnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_BNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SBnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_BNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c new file mode 100644 index 0000000000..9e7cdbf09e --- /dev/null +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndConsumer.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndTopic.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "tname.h" + +#define MND_CONSUMER_VER_NUMBER 1 +#define MND_CONSUMER_RESERVE_SIZE 64 + +static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer); +static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); +static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer); +static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer); +static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pConsumer, SConsumerObj *pNewConsumer); +static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); + +static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); + +int32_t mndInitConsumer(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_CONSUMER, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndConsumerActionEncode, + .decodeFp = (SdbDecodeFp)mndConsumerActionDecode, + .insertFp = (SdbInsertFp)mndConsumerActionInsert, + .updateFp = (SdbUpdateFp)mndConsumerActionUpdate, + .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp); + mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq); + mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupConsumer(SMnode *pMnode) {} + +static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer) { + int32_t size = sizeof(SConsumerObj) + MND_CONSUMER_RESERVE_SIZE; + SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pConsumer->name, TSDB_TABLE_FNAME_LEN); + SDB_SET_BINARY(pRaw, dataPos, pConsumer->db, TSDB_DB_FNAME_LEN); + SDB_SET_INT64(pRaw, dataPos, pConsumer->createTime); + SDB_SET_INT64(pRaw, dataPos, pConsumer->updateTime); + SDB_SET_INT64(pRaw, dataPos, pConsumer->uid); + /*SDB_SET_INT64(pRaw, dataPos, pConsumer->dbUid);*/ + SDB_SET_INT32(pRaw, dataPos, pConsumer->version); + + SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE); + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != MND_CONSUMER_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode consumer since %s", terrstr()); + return NULL; + } + + int32_t size = sizeof(SConsumerObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + SConsumerObj *pConsumer = sdbGetRowObj(pRow); + if (pConsumer == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumer->name, TSDB_TABLE_FNAME_LEN); + SDB_GET_BINARY(pRaw, pRow, dataPos, pConsumer->db, TSDB_DB_FNAME_LEN); + SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->createTime); + SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->updateTime); + SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->uid); + /*SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->dbUid);*/ + SDB_GET_INT32(pRaw, pRow, dataPos, &pConsumer->version); + + SDB_GET_RESERVE(pRaw, pRow, dataPos, MND_CONSUMER_RESERVE_SIZE); + + return pRow; +} + +static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer) { + mTrace("consumer:%s, perform insert action", pConsumer->name); + return 0; +} + +static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer) { + mTrace("consumer:%s, perform delete action", pConsumer->name); + return 0; +} + +static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pOldConsumer, SConsumerObj *pNewConsumer) { + mTrace("consumer:%s, perform update action", pOldConsumer->name); + atomic_exchange_32(&pOldConsumer->updateTime, pNewConsumer->updateTime); + atomic_exchange_32(&pOldConsumer->version, pNewConsumer->version); + + taosWLockLatch(&pOldConsumer->lock); + + // TODO handle update + + taosWUnLockLatch(&pOldConsumer->lock); + return 0; +} + +SConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) { + SSdb *pSdb = pMnode->pSdb; + SConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); + if (pConsumer == NULL) { + /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ + } + return pConsumer; +} + +void mndReleaseConsumer(SMnode *pMnode, SConsumerObj *pConsumer) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pConsumer); +} + +static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SCMSubscribeReq *pSubscribe; + tDeserializeSCMSubscribeReq(msgStr, pSubscribe); + // add consumerGroupId -> list to sdb + // add consumerId -> list to sdb + // add consumer -> list to sdb + return 0; +} + +static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + + mDebug("consumer:%s, start to retrieve meta", pInfo->tableFname); + +#if 0 + SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("consumer:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + SConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pInfo->tableFname); + if (pConsumer == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_CONSUMER; + mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + taosRLockLatch(&pConsumer->lock); + int32_t totalCols = pConsumer->numOfColumns + pConsumer->numOfTags; + int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + + STableMetaMsg *pMeta = rpcMallocCont(contLen); + if (pMeta == NULL) { + taosRUnLockLatch(&pConsumer->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + return -1; + } + + memcpy(pMeta->consumerFname, pConsumer->name, TSDB_TABLE_FNAME_LEN); + pMeta->numOfTags = htonl(pConsumer->numOfTags); + pMeta->numOfColumns = htonl(pConsumer->numOfColumns); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->update = pDb->cfg.update; + pMeta->sversion = htonl(pConsumer->version); + pMeta->tuid = htonl(pConsumer->uid); + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pSrcSchema = &pConsumer->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + taosRUnLockLatch(&pConsumer->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseConsumer(pMnode, pConsumer); + + pMsg->pCont = pMeta; + pMsg->contLen = contLen; + + mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags); +#endif + return 0; +} + +static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumOfConsumers) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfConsumers = 0; + void *pIter = NULL; + while (1) { + SConsumerObj *pConsumer = NULL; + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) break; + + if (strcmp(pConsumer->db, dbName) == 0) { + numOfConsumers++; + } + + sdbRelease(pSdb, pConsumer); + } + + *pNumOfConsumers = numOfConsumers; + return 0; +} + +static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfConsumers(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "tags"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_CONSUMER); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SConsumerObj *pConsumer = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; + + tstrncpy(prefix, pShow->db, 64); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = (int32_t)strlen(prefix); + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer); + if (pShow->pIter == NULL) break; + + if (strncmp(pConsumer->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pConsumer); + continue; + } + + cols = 0; + + char consumerName[TSDB_TABLE_NAME_LEN] = {0}; + tstrncpy(consumerName, pConsumer->name + prefixLen, TSDB_TABLE_NAME_LEN); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, consumerName); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pConsumer->createTime; + cols++; + + /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ + /**(int32_t *)pWrite = pConsumer->numOfColumns;*/ + /*cols++;*/ + + /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ + /**(int32_t *)pWrite = pConsumer->numOfTags;*/ + /*cols++;*/ + + numOfRows++; + sdbRelease(pSdb, pConsumer); + } + + pShow->numOfReads += numOfRows; + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d05e301024..f9199f9eed 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -549,7 +549,7 @@ static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pO return 0; } -static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { SSdbRaw *pCommitRaw = mndDbActionEncode(pNewDb); if (pCommitRaw == NULL) return -1; if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; @@ -725,7 +725,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; - SVnodeGid * pVgid = pVgroup->vnodeGid + vn; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode == NULL) return -1; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 43b458a52a..1c7f8544e9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -40,8 +40,6 @@ static const char *offlineReason[] = { "unknown", }; -static const char *dnodeStatus[] = {"offline", "ready", "creating", "dropping"}; - static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -198,6 +196,8 @@ static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { sdbCancelFetch(pSdb, pIter); return pDnode; } + + sdbRelease(pSdb, pDnode); } return NULL; @@ -208,10 +208,12 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } -bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { - int64_t ms = taosGetTimestampMs(); - int64_t interval = ABS(pDnode->lastAccessTime - ms); +bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { + int64_t interval = ABS(pDnode->lastAccessTime - curMs); if (interval > 3500 * pMnode->cfg.statusInterval) { + if (pDnode->rebootTime > 0) { + pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; + } return false; } return true; @@ -278,8 +280,8 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->clusterId = htobe64(pStatus->clusterId); pStatus->rebootTime = htobe64(pStatus->rebootTime); pStatus->updateTime = htobe64(pStatus->updateTime); - pStatus->numOfCores = htons(pStatus->numOfCores); - pStatus->numOfSupportVnodes = htons(pStatus->numOfSupportVnodes); + pStatus->numOfCores = htonl(pStatus->numOfCores); + pStatus->numOfSupportVnodes = htonl(pStatus->numOfSupportVnodes); pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval); pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } @@ -287,96 +289,99 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SStatusMsg *pStatus = pMsg->rpcMsg.pCont; + SDnodeObj *pDnode = NULL; + int32_t code = -1; + mndParseStatusMsg(pStatus); - SDnodeObj *pDnode = NULL; if (pStatus->dnodeId == 0) { pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp); if (pDnode == NULL) { mDebug("dnode:%s, not created yet", pStatus->dnodeEp); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto PROCESS_STATUS_MSG_OVER; } } else { pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId); if (pDnode == NULL) { pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp); - if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { + if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_DNODE_ID_NOT_MATCH; } mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp); - mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - return -1; + goto PROCESS_STATUS_MSG_OVER; } } - if (pStatus->sver != pMnode->cfg.sver) { - if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { - pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; - } - mndReleaseDnode(pMnode, pDnode); - mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); - terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; - return -1; - } + int64_t curMs = taosGetTimestampMs(); + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + bool needCheckCfg = !(online && pDnode->rebootTime == pStatus->rebootTime); - if (pStatus->dnodeId == 0) { - mDebug("dnode:%d %s, first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); - } else { - if (pStatus->clusterId != pMnode->clusterId) { - if (pDnode != NULL && pDnode->status != DND_STATUS_READY) { - pDnode->offlineReason = DND_REASON_CLUSTER_ID_NOT_MATCH; + if (needCheckCfg) { + if (pStatus->sver != pMnode->cfg.sver) { + if (pDnode != NULL) { + pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; + } + mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver); + terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; + goto PROCESS_STATUS_MSG_OVER; + } + + if (pStatus->dnodeId == 0) { + mDebug("dnode:%d %s, first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); + } else { + if (pStatus->clusterId != pMnode->clusterId) { + if (pDnode != NULL) { + pDnode->offlineReason = DND_REASON_CLUSTER_ID_NOT_MATCH; + } + mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId, + pMnode->clusterId); + terrno = TSDB_CODE_MND_INVALID_CLUSTER_ID; + goto PROCESS_STATUS_MSG_OVER; + } else { + pDnode->accessTimes++; + mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } - mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId, - pMnode->clusterId); - mndReleaseDnode(pMnode, pDnode); - terrno != TSDB_CODE_MND_INVALID_CLUSTER_ID; - return -1; - } else { - pDnode->accessTimes++; - mTrace("dnode:%d, status received, access times %d", pDnode->id, pDnode->accessTimes); } - } - if (pDnode->status == DND_STATUS_OFFLINE) { // Verify whether the cluster parameters are consistent when status change from offline to ready int32_t ret = mndCheckClusterCfgPara(pMnode, &pStatus->clusterCfg); if (0 != ret) { pDnode->offlineReason = ret; mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[ret]); - mndReleaseDnode(pMnode, pDnode); terrno = TSDB_CODE_MND_INVALID_CLUSTER_CFG; - return -1; + goto PROCESS_STATUS_MSG_OVER; } mInfo("dnode:%d, from offline to online", pDnode->id); + + pDnode->rebootTime = pStatus->rebootTime; + pDnode->numOfCores = pStatus->numOfCores; + pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; + + int32_t numOfEps = mndGetDnodeSize(pMnode); + int32_t contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp); + SStatusRsp *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto PROCESS_STATUS_MSG_OVER; + } + + pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); + pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); + mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); + + pMsg->contLen = contLen; + pMsg->pCont = pRsp; } - pDnode->rebootTime = pStatus->rebootTime; - pDnode->numOfCores = pStatus->numOfCores; - pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes; - pDnode->lastAccessTime = taosGetTimestampMs(); - pDnode->status = DND_STATUS_READY; + pDnode->lastAccessTime = curMs; + code = 0; - int32_t numOfEps = mndGetDnodeSize(pMnode); - int32_t contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp); - SStatusRsp *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - mndReleaseDnode(pMnode, pDnode); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pRsp->dnodeCfg.dnodeId = htonl(pDnode->id); - pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId); - mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps); - - pMsg->contLen = contLen; - pMsg->pCont = pRsp; +PROCESS_STATUS_MSG_OVER: mndReleaseDnode(pMnode, pDnode); - - return 0; + return code; } static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { @@ -416,10 +421,9 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg * static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; - + pCreate->port = htonl(pCreate->port); mDebug("dnode:%s:%d, start to create", pCreate->fqdn, pCreate->port); - pCreate->port = htonl(pCreate->port); if (pCreate->fqdn[0] == 0 || pCreate->port <= 0 || pCreate->port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr()); @@ -638,7 +642,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "max_vnodes"); + strcpy(pSchema[cols].name, "support_vnodes"); pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; @@ -682,10 +686,12 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i int32_t cols = 0; SDnodeObj *pDnode = NULL; char *pWrite; + int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); if (pShow->pIter == NULL) break; + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); cols = 0; @@ -706,8 +712,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - const char *status = dnodeStatus[pDnode->status]; - STR_TO_VARSTR(pWrite, status); + STR_TO_VARSTR(pWrite, online ? "ready" : "offline"); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -715,11 +720,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pDnode->status == DND_STATUS_READY) { - STR_TO_VARSTR(pWrite, ""); - } else { - STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); - } + STR_TO_VARSTR(pWrite, online ? "" : offlineReason[pDnode->offlineReason]); cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1800fd8e83..ad3d5e1cf6 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -219,6 +219,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { } pEpSet->numOfEps++; + sdbRelease(pSdb, pObj); } } @@ -251,7 +252,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno void *pIter = NULL; int32_t numOfReplicas = 0; - SCreateMnodeInMsg createMsg = {0}; + SDCreateMnodeMsg createMsg = {0}; while (1) { SMnodeObj *pMObj = NULL; pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); @@ -281,18 +282,18 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno STransAction action = {0}; - SAlterMnodeInMsg *pMsg = malloc(sizeof(SAlterMnodeInMsg)); + SDAlterMnodeMsg *pMsg = malloc(sizeof(SDAlterMnodeMsg)); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; } - memcpy(pMsg, &createMsg, sizeof(SAlterMnodeInMsg)); + memcpy(pMsg, &createMsg, sizeof(SDAlterMnodeMsg)); pMsg->dnodeId = htonl(pMObj->id); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pMsg; - action.contLen = sizeof(SAlterMnodeInMsg); + action.contLen = sizeof(SDAlterMnodeMsg); action.msgType = TDMT_DND_ALTER_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -309,14 +310,14 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - SCreateMnodeInMsg *pMsg = malloc(sizeof(SCreateMnodeInMsg)); + SDCreateMnodeMsg *pMsg = malloc(sizeof(SDCreateMnodeMsg)); if (pMsg == NULL) return -1; - memcpy(pMsg, &createMsg, sizeof(SAlterMnodeInMsg)); + memcpy(pMsg, &createMsg, sizeof(SDAlterMnodeMsg)); pMsg->dnodeId = htonl(pObj->id); action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pMsg; - action.contLen = sizeof(SCreateMnodeInMsg); + action.contLen = sizeof(SDCreateMnodeMsg); action.msgType = TDMT_DND_CREATE_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -327,9 +328,9 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } -static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SCreateMnodeMsg *pCreate) { +static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateMnodeMsg *pCreate) { SMnodeObj mnodeObj = {0}; - mnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_MNODE); + mnodeObj.id = pDnode->id; mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.updateTime = mnodeObj.createdTime; @@ -370,7 +371,7 @@ CREATE_MNODE_OVER: static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont; + SMCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont; pCreate->dnodeId = htonl(pCreate->dnodeId); @@ -423,7 +424,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode void *pIter = NULL; int32_t numOfReplicas = 0; - SAlterMnodeInMsg alterMsg = {0}; + SDAlterMnodeMsg alterMsg = {0}; while (1) { SMnodeObj *pMObj = NULL; pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); @@ -449,18 +450,18 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (pMObj->id != pObj->id) { STransAction action = {0}; - SAlterMnodeInMsg *pMsg = malloc(sizeof(SAlterMnodeInMsg)); + SDAlterMnodeMsg *pMsg = malloc(sizeof(SDAlterMnodeMsg)); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pMObj); return -1; } - memcpy(pMsg, &alterMsg, sizeof(SAlterMnodeInMsg)); + memcpy(pMsg, &alterMsg, sizeof(SDAlterMnodeMsg)); pMsg->dnodeId = htonl(pMObj->id); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pMsg; - action.contLen = sizeof(SAlterMnodeInMsg); + action.contLen = sizeof(SDAlterMnodeMsg); action.msgType = TDMT_DND_ALTER_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -478,7 +479,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - SDropMnodeInMsg *pMsg = malloc(sizeof(SDropMnodeInMsg)); + SDDropMnodeMsg *pMsg = malloc(sizeof(SDDropMnodeMsg)); if (pMsg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -487,7 +488,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pMsg; - action.contLen = sizeof(SDropMnodeInMsg); + action.contLen = sizeof(SDDropMnodeMsg); action.msgType = TDMT_DND_DROP_MNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); @@ -537,7 +538,7 @@ DROP_MNODE_OVER: static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont; + SMDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont; pDrop->dnodeId = htonl(pDrop->dnodeId); mDebug("mnode:%d, start to drop", pDrop->dnodeId); @@ -562,7 +563,7 @@ static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg) { return -1; } - sdbRelease(pMnode->pSdb, pMnode); + sdbRelease(pMnode->pSdb, pObj); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c new file mode 100644 index 0000000000..3b27764c7c --- /dev/null +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndQnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_QNODE_VER_NUMBER 1 +#define TSDB_QNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj); +static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); +static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode); +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); + +int32_t mndInitQnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_QNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, + .insertFp = (SdbInsertFp)mndQnodeActionInsert, + .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndQnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_QNODE, mndProcessCreateQnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_QNODE, mndProcessDropQnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndProcessCreateQnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndProcessDropQnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndGetQnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndRetrieveQnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndCancelGetNextQnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupQnode(SMnode *pMnode) {} + +static SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { + SSdb *pSdb = pMnode->pSdb; + SQnodeObj *pObj = sdbAcquire(pSdb, SDB_QNODE, &qnodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_QNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_QNODE, TSDB_QNODE_VER_NUMBER, sizeof(SQnodeObj) + TSDB_QNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_QNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_QNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode qnode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SQnodeObj)); + SQnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_QNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) { + mTrace("qnode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { + mTrace("qnode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode) { + mTrace("qnode:%d, perform update action", pOldQnode->id); + pOldQnode->updateTime = pNewQnode->updateTime; + return 0; +} + +static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDCreateQnodeMsg *pMsg = malloc(sizeof(SDCreateQnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateQnodeMsg); + action.msgType = TDMT_DND_CREATE_QNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateQnodeMsg *pCreate) { + SQnodeObj qnodeObj = {0}; + qnodeObj.id = pDnode->id; + qnodeObj.createdTime = taosGetTimestampMs(); + qnodeObj.updateTime = qnodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_QNODE_OVER; + } + mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + code = 0; + +CREATE_QNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateQnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("qnode:%d, start to create", pCreate->dnodeId); + + SQnodeObj *pObj = mndAcquireQnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("qnode:%d, qnode already exist", pObj->id); + mndReleaseQnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("qnode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateQnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDDropQnodeMsg *pMsg = malloc(sizeof(SDDropQnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropQnodeMsg); + action.msgType = TDMT_DND_DROP_QNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pMsg, SQnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("qnode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_QNODE_OVER; + } + + mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); + + if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + code = 0; + +DROP_QNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMDropQnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("qnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("qnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SQnodeObj *pObj = mndAcquireQnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("qnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropQnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_QNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SQnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_QNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index b89de94452..d4b4459176 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -20,7 +20,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); -static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); +static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); @@ -52,8 +52,8 @@ void mndCleanupShow(SMnode *pMnode) { static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { SShowMgmt *pMgmt = &pMnode->showMgmt; - int32_t showId = atomic_add_fetch_32(&pMgmt->showId, 1); - if (showId == 0) atomic_add_fetch_32(&pMgmt->showId, 1); + int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1); + if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1); int32_t size = sizeof(SShowObj) + pMsg->payloadLen; SShowObj showObj = {0}; @@ -65,14 +65,14 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) { memcpy(showObj.payload, pMsg->payload, pMsg->payloadLen); int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; - SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), &showObj, size, keepTime); + SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to put into cache since %s", showId, terrstr()); + mError("show:0x%"PRIx64", failed to put into cache since %s", showId, terrstr()); return NULL; } - mTrace("show:%d, is created, data:%p", showId, pShow); + mTrace("show:0x%"PRIx64", is created, data:%p", showId, pShow); return pShow; } @@ -87,25 +87,25 @@ static void mndFreeShowObj(SShowObj *pShow) { } } - mTrace("show:%d, is destroyed, data:%p", pShow->id, pShow); + mTrace("show:0x%d, is destroyed, data:%p", pShow->id, pShow); } -static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) { +static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId) { SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowObj *pShow = taosCacheAcquireByKey(pMgmt->cache, &showId, sizeof(int32_t)); + SShowObj *pShow = taosCacheAcquireByKey(pMgmt->cache, &showId, sizeof(showId)); if (pShow == NULL) { - mError("show:%d, already destroyed", showId); + mError("show:0x%"PRIx64", already destroyed", showId); return NULL; } - mTrace("show:%d, acquired from cache, data:%p", pShow->id, pShow); + mTrace("show:0x%"PRIx64", acquired from cache, data:%p", pShow->id, pShow); return pShow; } static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { if (pShow == NULL) return; - mTrace("show:%d, released from cache, data:%p force:%d", pShow->id, pShow, forceRemove); + mTrace("show:0x%"PRIx64", released from cache, data:%p force:%d", pShow->id, pShow, forceRemove); // A bug in tcache.c forceRemove = 0; @@ -146,18 +146,18 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { if (pRsp == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type)); + mError("show:0x%"PRIx64", failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type)); return -1; } int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta); - mDebug("show:%d, get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow->numOfRows, + mDebug("show:0x%"PRIx64", get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow->numOfRows, pShow->numOfColumns, mndShowStr(type), tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; - pMnodeMsg->pCont = pRsp; - pRsp->showId = htonl(pShow->id); + pMnodeMsg->pCont = pRsp; + pRsp->showId = htobe64(pShow->id); mndReleaseShowObj(pShow, false); return TSDB_CODE_SUCCESS; } else { @@ -175,7 +175,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { int32_t rowsRead = 0; SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont; - int32_t showId = htonl(pRetrieve->showId); + int64_t showId = htobe64(pRetrieve->showId); SShowObj *pShow = mndAcquireShowObj(pMnode, showId); if (pShow == NULL) { @@ -188,15 +188,15 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); terrno = TSDB_CODE_MSG_NOT_PROCESSED; - mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:0x%"PRIx64", failed to retrieve data since %s", pShow->id, terrstr()); return -1; } - mDebug("show:%d, start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads, + mDebug("show:0x%"PRIx64", start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads, pShow->numOfRows, mndShowStr(pShow->type)); if (mndCheckRetrieveFinished(pShow)) { - mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); + mDebug("show:0x%"PRIx64", read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); pShow->numOfReads = pShow->numOfRows; } @@ -219,7 +219,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (pRsp == NULL) { mndReleaseShowObj(pShow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + mError("show:0x%"PRIx64", failed to retrieve data since %s", pShow->id, terrstr()); return -1; } @@ -228,7 +228,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); } - mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); + mDebug("show:0x%"PRIx64", stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); pRsp->numOfRows = htonl(rowsRead); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision @@ -238,10 +238,10 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { pRsp->completed = 1; - mDebug("show:%d, retrieve completed", pShow->id); + mDebug("show:0x%"PRIx64", retrieve completed", pShow->id); mndReleaseShowObj(pShow, true); } else { - mDebug("show:%d, retrieve not completed yet", pShow->id); + mDebug("show:0x%"PRIx64", retrieve not completed yet", pShow->id); mndReleaseShowObj(pShow, false); } @@ -262,6 +262,12 @@ char *mndShowStr(int32_t showType) { return "show dnodes"; case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; + case TSDB_MGMT_TABLE_QNODE: + return "show qnodes"; + case TSDB_MGMT_TABLE_SNODE: + return "show snodes"; + case TSDB_MGMT_TABLE_BNODE: + return "show bnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; case TSDB_MGMT_TABLE_STB: diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c new file mode 100644 index 0000000000..a4825c0e82 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndSnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_SNODE_VER_NUMBER 1 +#define TSDB_SNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj); +static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); +static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode); +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); + +int32_t mndInitSnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, + .insertFp = (SdbInsertFp)mndSnodeActionInsert, + .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndSnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SNODE, mndProcessCreateSnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SNODE, mndProcessDropSnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndProcessCreateSnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndProcessDropSnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndGetSnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndRetrieveSnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndCancelGetNextSnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupSnode(SMnode *pMnode) {} + +static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { + SSdb *pSdb = pMnode->pSdb; + SSnodeObj *pObj = sdbAcquire(pSdb, SDB_SNODE, &snodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_SNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_SNODE, TSDB_SNODE_VER_NUMBER, sizeof(SSnodeObj) + TSDB_SNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_SNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_SNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode snode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SSnodeObj)); + SSnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_SNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj) { + mTrace("snode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("snode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj) { + mTrace("snode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode) { + mTrace("snode:%d, perform update action", pOldSnode->id); + pOldSnode->updateTime = pNewSnode->updateTime; + return 0; +} + +static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDCreateSnodeMsg *pMsg = malloc(sizeof(SDCreateSnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateSnodeMsg); + action.msgType = TDMT_DND_CREATE_SNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateSnodeMsg *pCreate) { + SSnodeObj snodeObj = {0}; + snodeObj.id = pDnode->id; + snodeObj.createdTime = taosGetTimestampMs(); + snodeObj.updateTime = snodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_SNODE_OVER; + } + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + code = 0; + +CREATE_SNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateSnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("snode:%d, start to create", pCreate->dnodeId); + + SSnodeObj *pObj = mndAcquireSnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("snode:%d, snode already exist", pObj->id); + mndReleaseSnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("snode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateSnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDDropSnodeMsg *pMsg = malloc(sizeof(SDDropSnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropSnodeMsg); + action.msgType = TDMT_DND_DROP_SNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pMsg, SSnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_SNODE_OVER; + } + + mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + + if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + code = 0; + +DROP_SNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMDropSnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("snode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SSnodeObj *pObj = mndAcquireSnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("snode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropSnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_SNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SSnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_SNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index af18c814ca..f81a19a780 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -201,14 +201,16 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { } static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int *pContLen) { -#if 1 SVCreateTbReq req; void * buf; int bsize; SMsgHead * pMsgHead; req.ver = 0; - req.name = pStb->name; + SName name = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE); + + req.name = (char*) tNameGetTableName(&name); req.ttl = 0; req.keep = 0; req.type = TD_SUPER_TABLE; @@ -235,43 +237,12 @@ static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb *pContLen = sizeof(SMsgHead) + bsize; return buf; - -#else - int32_t totalCols = pStb->numOfTags + pStb->numOfColumns; - int32_t contLen = totalCols * sizeof(SSchema) + sizeof(SCreateStbInternalMsg); - - SCreateStbInternalMsg *pCreate = calloc(1, contLen); - if (pCreate == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pCreate->head.contLen = htonl(contLen); - pCreate->head.vgId = htonl(pVgroup->vgId); - memcpy(pCreate->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pCreate->suid = htobe64(pStb->uid); - pCreate->sverson = htonl(pStb->version); - pCreate->ttl = 0; - pCreate->keep = 0; - pCreate->numOfTags = htonl(pStb->numOfTags); - pCreate->numOfColumns = htonl(pStb->numOfColumns); - - memcpy(pCreate->pSchema, pStb->pSchema, totalCols * sizeof(SSchema)); - for (int32_t t = 0; t < totalCols; ++t) { - SSchema *pSchema = &pCreate->pSchema[t]; - pSchema->bytes = htonl(pSchema->bytes); - pSchema->colId = htonl(pSchema->colId); - } - - *pContLen = contLen; - return pCreate; -#endif } -static SDropStbInternalMsg *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SDropStbInternalMsg); +static SVDropStbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t contLen = sizeof(SVDropStbReq); - SDropStbInternalMsg *pDrop = calloc(1, contLen); + SVDropStbReq *pDrop = calloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -402,7 +373,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SDropStbInternalMsg *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); + SVDropStbReq *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); if (pMsg == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -413,7 +384,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pMsg; - action.contLen = sizeof(SDropStbInternalMsg); + action.contLen = sizeof(SVDropStbReq); action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 597142d326..24e32e07b4 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -33,10 +33,7 @@ static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic); static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pTopic, STopicObj *pNewTopic); static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); -static int32_t mndProcessAlterTopicMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); -static int32_t mndProcessCreateTopicInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessAlterTopicInRsp(SMnodeMsg *pMsg); static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); @@ -53,19 +50,8 @@ int32_t mndInitTopic(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTopicActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicMsg); - mndSetMsgHandle(pMnode, TDMT_MND_ALTER_TOPIC, mndProcessAlterTopicMsg); mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicMsg); - mndSetMsgHandle(pMnode, TDMT_VND_CREATE_TOPIC_RSP, mndProcessCreateTopicInRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_TOPIC_RSP, mndProcessAlterTopicInRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); - mndSetMsgHandle(pMnode, TDMT_VND_TABLE_META, mndProcessTopicMetaMsg); - - /*mndAddShowMetaHandle(pMnode, TSDB_MGMT_TOPIC, mndGetTopicMeta);*/ - /*mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TOPIC, mndRetrieveTopic);*/ - /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TOPIC, mndCancelGetNextTopic);*/ - - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicMsg); - mndSetMsgHandle(pMnode, TDMT_VND_CREATE_TOPIC_RSP, mndProcessCreateTopicInRsp); return sdbSetTable(pMnode->pSdb, table); } @@ -145,24 +131,9 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pOldTopic, STopicObj atomic_exchange_32(&pOldTopic->version, pNewTopic->version); taosWLockLatch(&pOldTopic->lock); -#if 0 + + //TODO handle update - pOldTopic->numOfColumns = pNewTopic->numOfColumns; - pOldTopic->numOfTags = pNewTopic->numOfTags; - int32_t totalCols = pNewTopic->numOfTags + pNewTopic->numOfColumns; - int32_t totalSize = totalCols * sizeof(SSchema); - - if (pOldTopic->numOfTags + pOldTopic->numOfColumns < totalCols) { - void *pSchema = malloc(totalSize); - if (pSchema != NULL) { - free(pOldTopic->pSchema); - pOldTopic->pSchema = pSchema; - } - } - - memcpy(pOldTopic->pSchema, pNewTopic->pSchema, totalSize); - -#endif taosWUnLockLatch(&pOldTopic->lock); return 0; } @@ -191,45 +162,10 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { return mndAcquireDb(pMnode, db); } -static SCreateTopicInternalMsg *mndBuildCreateTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { - int32_t totalCols = 0; - int32_t contLen = sizeof(SCreateTopicInternalMsg) + pTopic->execLen + pTopic->sqlLen; +static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { + int32_t contLen = sizeof(SDDropTopicMsg); - SCreateTopicInternalMsg *pCreate = calloc(1, contLen); - if (pCreate == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pCreate->head.contLen = htonl(contLen); - pCreate->head.vgId = htonl(pVgroup->vgId); - memcpy(pCreate->name, pTopic->name, TSDB_TABLE_FNAME_LEN); - pCreate->tuid = htobe64(pTopic->uid); - pCreate->sverson = htonl(pTopic->version); - - pCreate->sql = malloc(pTopic->sqlLen); - if (pCreate->sql == NULL) { - free(pCreate); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - memcpy(pCreate->sql, pTopic->sql, pTopic->sqlLen); - - pCreate->executor = malloc(pTopic->execLen); - if (pCreate->executor == NULL) { - free(pCreate); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - memcpy(pCreate->executor, pTopic->executor, pTopic->execLen); - - return pCreate; -} - -static SDropTopicInternalMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { - int32_t contLen = sizeof(SDropTopicInternalMsg); - - SDropTopicInternalMsg *pDrop = calloc(1, contLen); + SDDropTopicMsg *pDrop = calloc(1, contLen); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -243,109 +179,12 @@ static SDropTopicInternalMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgro return pDrop; } -static int32_t mndCheckCreateTopicMsg(SCreateTopicMsg *pCreate) { +static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *pCreate) { // deserialize and other stuff return 0; } -static int32_t mndSetCreateTopicRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { - SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; - - return 0; -} - -static int32_t mndSetCreateTopicUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { - SSdbRaw *pUndoRaw = mndTopicActionEncode(pTopic); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - - return 0; -} - -static int32_t mndSetCreateTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { - SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - -static int32_t mndSetCreateTopicRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { - SSdb *pSdb = pMnode->pSdb; - SVgObj *pVgroup = NULL; - void *pIter = NULL; - - while (1) { - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - if (pVgroup->dbUid != pDb->uid) continue; - - SCreateTopicInternalMsg *pMsg = mndBuildCreateTopicMsg(pMnode, pVgroup, pTopic); - if (pMsg == NULL) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgroup); - action.pCont = pMsg; - action.contLen = htonl(pMsg->head.contLen); - action.msgType = TDMT_VND_CREATE_TOPIC; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(pMsg); - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - return -1; - } - sdbRelease(pSdb, pVgroup); - } - - return 0; -} - -static int32_t mndSetCreateTopicUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, STopicObj *pTopic) { - SSdb *pSdb = pMnode->pSdb; - SVgObj *pVgroup = NULL; - void *pIter = NULL; - - while (1) { - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - if (pVgroup->dbUid != pDb->uid) continue; - - SDropTopicInternalMsg *pMsg = mndBuildDropTopicMsg(pMnode, pVgroup, pTopic); - if (pMsg == NULL) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgroup); - action.pCont = pMsg; - action.contLen = sizeof(SDropTopicInternalMsg); - action.msgType = TDMT_VND_DROP_TOPIC; - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - free(pMsg); - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pVgroup); - return -1; - } - sdbRelease(pSdb, pVgroup); - } - - return 0; -} - -static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCreateTopicMsg *pCreate, SDbObj *pDb) { +static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { STopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -355,66 +194,17 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCreateTopicMsg * topicObj.dbUid = pDb->uid; topicObj.version = 1; -#if 0 - int32_t totalCols = topicObj.numOfColumns + topicObj.numOfTags; - int32_t totalSize = totalCols * sizeof(SSchema); - topicObj.sql = malloc(totalSize); - if (topicObj.sql == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - memcpy(topicObj.sql, pCreate->sql, totalSize); -#endif - - int32_t code = 0; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } - mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); - - if (mndSetCreateTopicRedoLogs(pMnode, pTrans, pDb, &topicObj) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto CREATE_TOPIC_OVER; - } - - if (mndSetCreateTopicUndoLogs(pMnode, pTrans, pDb, &topicObj) != 0) { - mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); - goto CREATE_TOPIC_OVER; - } - - if (mndSetCreateTopicCommitLogs(pMnode, pTrans, pDb, &topicObj) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto CREATE_TOPIC_OVER; - } - - if (mndSetCreateTopicRedoActions(pMnode, pTrans, pDb, &topicObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_TOPIC_OVER; - } - - if (mndSetCreateTopicUndoActions(pMnode, pTrans, pDb, &topicObj) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto CREATE_TOPIC_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } - - code = 0; - -CREATE_TOPIC_OVER: - mndTransDrop(pTrans); - return code; + SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); + if (pTopicRaw == NULL) return -1; + if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; + return sdbWrite(pMnode->pSdb, pTopicRaw); } static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SCreateTopicMsg *pCreate = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + char *msgStr = pMsg->rpcMsg.pCont; + SCMCreateTopicReq* pCreate; + tDeserializeSCMCreateTopicReq(msgStr, pCreate); mDebug("topic:%s, start to create", pCreate->name); @@ -436,15 +226,6 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { } } - // topic should have different name with stb - SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); - if (pStb != NULL) { - sdbRelease(pMnode->pSdb, pStb); - terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_STB; - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } - SDbObj *pDb = mndAcquireDbByTopic(pMnode, pCreate->name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -464,144 +245,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndCheckAlterTopicMsg(SAlterTopicMsg *pAlter) { - SSchema *pSchema = &pAlter->schema; - pSchema->colId = htonl(pSchema->colId); - pSchema->bytes = htonl(pSchema->bytes); - - if (pSchema->type <= 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; - return -1; - } - if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; - return -1; - } - if (pSchema->bytes <= 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; - return -1; - } - if (pSchema->name[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; - return -1; - } - - return 0; -} - -static int32_t mndUpdateTopic(SMnode *pMnode, SMnodeMsg *pMsg, STopicObj *pOldTopic, STopicObj *pNewTopic) { return 0; } - -static int32_t mndProcessAlterTopicMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SAlterTopicMsg *pAlter = pMsg->rpcMsg.pCont; - - mDebug("topic:%s, start to alter", pAlter->name); - - if (mndCheckAlterTopicMsg(pAlter) != 0) { - mError("topic:%s, failed to alter since %s", pAlter->name, terrstr()); - return -1; - } - - STopicObj *pTopic = mndAcquireTopic(pMnode, pAlter->name); - if (pTopic == NULL) { - terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - mError("topic:%s, failed to alter since %s", pAlter->name, terrstr()); - return -1; - } - - STopicObj topicObj = {0}; - memcpy(&topicObj, pTopic, sizeof(STopicObj)); - - int32_t code = mndUpdateTopic(pMnode, pMsg, pTopic, &topicObj); - mndReleaseTopic(pMnode, pTopic); - - if (code != 0) { - mError("topic:%s, failed to alter since %s", pAlter->name, tstrerror(code)); - return code; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -} - -static int32_t mndProcessAlterTopicInRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); - return 0; -} - -static int32_t mndSetDropTopicRedoLogs(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { - SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; - - return 0; -} - -static int32_t mndSetDropTopicUndoLogs(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { - SSdbRaw *pUndoRaw = mndTopicActionEncode(pTopic); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - -static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { - SSdbRaw *pCommitRaw = mndTopicActionEncode(pTopic); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; - - return 0; -} - -static int32_t mndSetDropTopicRedoActions(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { return 0; } - -static int32_t mndSetDropTopicUndoActions(SMnode *pMnode, STrans *pTrans, STopicObj *pTopic) { return 0; } - static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, STopicObj *pTopic) { - int32_t code = -1; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); - return -1; - } - mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); - - if (mndSetDropTopicRedoLogs(pMnode, pTrans, pTopic) != 0) { - mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); - goto DROP_TOPIC_OVER; - } - - if (mndSetDropTopicUndoLogs(pMnode, pTrans, pTopic) != 0) { - mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); - goto DROP_TOPIC_OVER; - } - - if (mndSetDropTopicCommitLogs(pMnode, pTrans, pTopic) != 0) { - mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); - goto DROP_TOPIC_OVER; - } - - if (mndSetDropTopicRedoActions(pMnode, pTrans, pTopic) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_TOPIC_OVER; - } - - if (mndSetDropTopicUndoActions(pMnode, pTrans, pTopic) != 0) { - mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); - goto DROP_TOPIC_OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - goto DROP_TOPIC_OVER; - } - - code = 0; - -DROP_TOPIC_OVER: - mndTransDrop(pTrans); return 0; } @@ -705,11 +349,6 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCreateTopicInRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); - return 0; -} - static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9263fca695..cf434dda39 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -79,17 +79,17 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { for (int32_t i = 0; i < redoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); - rawDataLen += sdbGetRawTotalSize(pTmp); + rawDataLen += (sdbGetRawTotalSize(pTmp) + 4); } for (int32_t i = 0; i < undoLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); - rawDataLen += sdbGetRawTotalSize(pTmp); + rawDataLen += (sdbGetRawTotalSize(pTmp) + 4); } for (int32_t i = 0; i < commitLogNum; ++i) { SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); - rawDataLen += sdbGetRawTotalSize(pTmp); + rawDataLen += (sdbGetRawTotalSize(pTmp) + 4); } for (int32_t i = 0; i < redoActionNum; ++i) { @@ -437,7 +437,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { - mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to encode while sync trans since %s", pTrans->id, terrstr()); return -1; } sdbSetRawStatus(pRaw, SDB_STATUS_READY); @@ -835,7 +835,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { - mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to encode while finish trans since %s", pTrans->id, terrstr()); } sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 06e62d2528..41de70283f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -120,6 +120,9 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SDB_GET_INT32(pRaw, pRow, dataPos, &pVgid->dnodeId) + if (pVgroup->replica == 1) { + pVgid->role = TAOS_SYNC_STATE_LEADER; + } } SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_VGROUP_RESERVE_SIZE) @@ -235,45 +238,55 @@ SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *p return pDrop; } +static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SDnodeObj *pDnode = pObj; + pDnode->numOfVnodes = 0; + return true; +} + +static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SDnodeObj *pDnode = pObj; + SArray *pArray = p1; + + pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); + + int64_t curMs = taosGetTimestampMs(); + bool online = mndIsDnodeOnline(pMnode, pDnode, curMs); + if (online && pDnode->numOfSupportVnodes > 0) { + taosArrayPush(pArray, pDnode); + } + + bool isMnode = mndIsMnode(pMnode, pDnode->id); + + mDebug("dnode:%d, vnodes:%d supportVnodes:%d isMnode:%d online:%d", pDnode->id, pDnode->numOfVnodes, + pDnode->numOfSupportVnodes, isMnode, online); + + if (isMnode) { + pDnode->numOfVnodes++; + } + + return true; +} + static SArray *mndBuildDnodesArray(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; int32_t numOfDnodes = mndGetDnodeSize(pMnode); + SArray *pArray = taosArrayInit(numOfDnodes, sizeof(SDnodeObj)); if (pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void *pIter = NULL; - while (1) { - SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); - if (pIter == NULL) break; - - int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); - - bool isMnode = mndIsMnode(pMnode, pDnode->id); - if (isMnode) { - pDnode->numOfVnodes++; - } - - bool isReady = mndIsDnodeInReadyStatus(pMnode, pDnode); - if (isReady) { - taosArrayPush(pArray, pDnode); - } - - mDebug("dnode:%d, numOfVnodes:%d numOfSupportVnodes:%d isMnode:%d ready:%d", pDnode->id, numOfVnodes, - pDnode->numOfSupportVnodes, isMnode, isReady); - sdbRelease(pSdb, pDnode); - } - + sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL); + sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, NULL, NULL); return pArray; } static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; - return d1Score > d2Score ? 0 : 1; + return d1Score > d2Score ? 1 : 0; } static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { @@ -298,7 +311,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr pVgid->role = TAOS_SYNC_STATE_FOLLOWER; } - mDebug("db:%s, vgId:%d, vindex:%d dnodeId:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); + mDebug("db:%s, vgId:%d, vn:%d dnode:%d is alloced", pVgroup->dbName, pVgroup->vgId, v, pVgid->dnodeId); pDnode->numOfVnodes++; } @@ -330,6 +343,8 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { uint32_t hashMax = UINT32_MAX; uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups; + if (maxVgId < 2) maxVgId = 2; + for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) { SVgObj *pVgroup = &pVgroups[v]; pVgroup->vgId = maxVgId++; @@ -406,6 +421,20 @@ static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SVgObj *pVgroup = pObj; + int64_t uid = *(int64_t *)p1; + int8_t *pReplica = p2; + int32_t *pNumOfVgroups = p3; + + if (pVgroup->dbUid == uid) { + *pReplica = MAX(*pReplica, pVgroup->replica); + (*pNumOfVgroups)++; + } + + return true; +} + static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); @@ -414,25 +443,10 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return -1; } - int8_t replica = 1; - int32_t numOfVgroups = 0; - - void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - if (pVgroup->dbUid == pDb->uid) { - replica = MAX(replica, pVgroup->replica); - numOfVgroups++; - } - - sdbRelease(pSdb, pVgroup); - } - - *pReplica = replica; - *pNumOfVgroups = numOfVgroups; + *pReplica = 1; + *pNumOfVgroups = 0; + sdbTraverse(pSdb, SDB_VGROUP, mndGetVgroupMaxReplicaFp, &pDb->uid, pReplica, pNumOfVgroups); + mndReleaseDb(pMnode, pDb); return 0; } @@ -534,25 +548,23 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { - SSdb *pSdb = pMnode->pSdb; - int32_t numOfVnodes = 0; - void *pIter = NULL; +static bool mndGetVnodesNumFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { + SVgObj *pVgroup = pObj; + int32_t dnodeId = *(int32_t *)p1; + int32_t *pNumOfVnodes = (int32_t *)p2; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { - numOfVnodes++; - } + for (int32_t v = 0; v < pVgroup->replica; ++v) { + if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { + (*pNumOfVnodes)++; } - - sdbRelease(pSdb, pVgroup); } + return true; +} + +int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { + int32_t numOfVnodes = 0; + sdbTraverse(pMnode->pSdb, SDB_VGROUP, mndGetVnodesNumFp, &dnodeId, &numOfVnodes, NULL); return numOfVnodes; } diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 84076f8c0a..e902e13a36 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -16,14 +16,16 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" #include "mndAuth.h" -#include "mndBalance.h" +#include "mndBnode.h" #include "mndCluster.h" #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" #include "mndProfile.h" +#include "mndQnode.h" #include "mndShow.h" +#include "mndSnode.h" #include "mndStb.h" #include "mndSync.h" #include "mndTelem.h" @@ -147,6 +149,9 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitSnode, mndCleanupSnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; @@ -162,7 +167,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; } if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-balance", mndInitBalance, mndCleanupBalance) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 78a90b9a7d..0388fa99f5 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -24,6 +24,12 @@ static const char *sdbTableName(ESdbType type) { return "cluster"; case SDB_MNODE: return "mnode"; + case SDB_QNODE: + return "qnode"; + case SDB_SNODE: + return "snode"; + case SDB_BNODE: + return "bnode"; case SDB_DNODE: return "dnode"; case SDB_USER: @@ -55,7 +61,8 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) { } else if (keyType == SDB_KEY_INT32) { mTrace("%s:%d, refCount:%d oper:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, oper); } else if (keyType == SDB_KEY_INT64) { - mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, oper); + mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, + oper); } else { } } @@ -329,6 +336,30 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter) { taosRUnLockLatch(pLock); } +void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3) { + SHashObj *hash = sdbGetHash(pSdb, type); + if (hash == NULL) return; + + SRWLatch *pLock = &pSdb->locks[type]; + taosRLockLatch(pLock); + + SSdbRow **ppRow = taosHashIterate(hash, NULL); + while (ppRow != NULL) { + SSdbRow *pRow = *ppRow; + if (pRow->status == SDB_STATUS_READY) { + bool isContinue = (*fp)(pSdb->pMnode, pRow->pObj, p1, p2, p3); + if (!isContinue) { + taosHashCancelIterate(hash, ppRow); + break; + } + } + + ppRow = taosHashIterate(hash, ppRow); + } + + taosRUnLockLatch(pLock); +} + int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return 0; diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index ff1a460228..8d2a72b268 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -24,12 +24,8 @@ void qndClose(SQnode *pQnode) { free(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } -int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } -int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - *pRsp = NULL; - return 0; -} diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 7ae4d49059..5d21a037e7 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -24,7 +24,7 @@ void sndClose(SSnode *pSnode) { free(pSnode); } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } -int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 78214ce14d..fdb9837292 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -31,10 +31,10 @@ #include "vnodeCommit.h" #include "vnodeFS.h" #include "vnodeMemAllocator.h" +#include "vnodeQuery.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" -#include "vnodeQuery.h" #ifdef __cplusplus extern "C" { @@ -62,6 +62,7 @@ typedef struct SVnodeMgr { extern SVnodeMgr vnodeMgr; struct SVnode { + int32_t vgId; char* path; SVnodeCfg config; SVState state; diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 1c6924040c..01df929080 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -16,6 +16,8 @@ #include "vnodeQuery.h" #include "vnodeDef.h" +static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); + int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { @@ -36,32 +38,132 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_DROP_TASK: return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_SHOW_TABLES: + return qWorkerProcessShowMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_SHOW_TABLES_FETCH: + return vnodeGetTableList(pVnode, pMsg); +// return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; - break; } - return 0; } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - STableInfoMsg *pReq = (STableInfoMsg *)(pMsg->pCont); - STableMetaMsg *pRspMsg; - int ret; + STableInfoMsg * pReq = (STableInfoMsg *)(pMsg->pCont); + STbCfg * pTbCfg = NULL; + STbCfg * pStbCfg = NULL; + tb_uid_t uid; + int32_t nCols; + int32_t nTagCols; + SSchemaWrapper *pSW; + STableMetaMsg * pTbMetaMsg; + SSchema * pTagSchema; - if (metaGetTableInfo(pVnode->pMeta, pReq->tableFname, &pRspMsg) < 0) { + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); + if (pTbCfg == NULL) { return -1; } - *pRsp = malloc(sizeof(SRpcMsg)); - if (TD_IS_NULL(*pRsp)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - free(pMsg); + if (pTbCfg->type == META_CHILD_TABLE) { + pStbCfg = metaGetTbInfoByUid(pVnode->pMeta, pTbCfg->ctbCfg.suid); + if (pStbCfg == NULL) { + return -1; + } + + pSW = metaGetTableSchema(pVnode->pMeta, pTbCfg->ctbCfg.suid, 0, true); + } else { + pSW = metaGetTableSchema(pVnode->pMeta, uid, 0, true); + } + + nCols = pSW->nCols; + if (pTbCfg->type == META_SUPER_TABLE) { + nTagCols = pTbCfg->stbCfg.nTagCols; + pTagSchema = pTbCfg->stbCfg.pTagSchema; + } else if (pTbCfg->type == META_SUPER_TABLE) { + nTagCols = pStbCfg->stbCfg.nTagCols; + pTagSchema = pStbCfg->stbCfg.pTagSchema; + } else { + nTagCols = 0; + pTagSchema = NULL; + } + + pTbMetaMsg = (STableMetaMsg *)calloc(1, sizeof(STableMetaMsg) + sizeof(SSchema) * (nCols + nTagCols)); + if (pTbMetaMsg == NULL) { return -1; } - // TODO - (*pRsp)->pCont = pRspMsg; + strcpy(pTbMetaMsg->tbFname, pTbCfg->name); + if (pTbCfg->type == META_CHILD_TABLE) { + strcpy(pTbMetaMsg->stbFname, pStbCfg->name); + pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); + } + pTbMetaMsg->numOfTags = htonl(nTagCols); + pTbMetaMsg->numOfColumns = htonl(nCols); + pTbMetaMsg->tableType = pTbCfg->type; + pTbMetaMsg->tuid = htobe64(uid); + pTbMetaMsg->vgId = htonl(pVnode->vgId); + + memcpy(pTbMetaMsg->pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols); + if (nTagCols) { + memcpy(POINTER_SHIFT(pTbMetaMsg->pSchema, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); + } + + for (int i = 0; i < nCols + nTagCols; i++) { + SSchema *pSch = pTbMetaMsg->pSchema + i; + pSch->colId = htonl(pSch->colId); + pSch->bytes = htonl(pSch->bytes); + } + return 0; +} + +/** + * @param pVnode + * @param pMsg + * @param pRsp + */ +static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { + SMTbCursor* pCur = metaOpenTbCursor(pVnode->pMeta); + SArray* pArray = taosArrayInit(10, POINTER_BYTES); + + char* name = NULL; + int32_t totalLen = 0; + while ((name = metaTbCursorNext(pCur)) != NULL) { + taosArrayPush(pArray, &name); + totalLen += strlen(name); + } + + metaCloseTbCursor(pCur); + + int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; + int32_t numOfTables = (int32_t) taosArrayGetSize(pArray); + + int32_t payloadLen = rowLen * numOfTables; +// SVShowTablesFetchReq *pFetchReq = pMsg->pCont; + + SVShowTablesFetchRsp *pFetchRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp) + payloadLen); + memset(pFetchRsp, 0, sizeof(struct SVShowTablesFetchRsp) + payloadLen); + + char* p = pFetchRsp->data; + for(int32_t i = 0; i < numOfTables; ++i) { + char* n = taosArrayGetP(pArray, i); + STR_TO_VARSTR(p, n); + + p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); + } + + pFetchRsp->numOfRows = htonl(numOfTables); + pFetchRsp->precision = 0; + + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pFetchRsp, + .contLen = sizeof(SVShowTablesFetchRsp) + payloadLen, + .code = 0, + }; + + rpcSendResponse(&rpcMsg); return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 51e3330ebf..af8af6a052 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -81,7 +81,7 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "meta.db", false) < 0) { + if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "schema.db", false) < 0) { metaCloseDB(pMeta); return -1; } @@ -378,10 +378,10 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { if (pTbCfg->type == META_SUPER_TABLE) { tsize += taosEncodeVariantU32(buf, pTbCfg->stbCfg.nTagCols); for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { - tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pSchema[i].type); - tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pSchema[i].colId); - tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pSchema[i].bytes); - tsize += taosEncodeString(buf, pTbCfg->stbCfg.pSchema[i].name); + tsize += taosEncodeFixedI8(buf, pTbCfg->stbCfg.pTagSchema[i].type); + tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].colId); + tsize += taosEncodeFixedI32(buf, pTbCfg->stbCfg.pTagSchema[i].bytes); + tsize += taosEncodeString(buf, pTbCfg->stbCfg.pTagSchema[i].name); } // tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); @@ -406,10 +406,10 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { buf = taosDecodeVariantU32(buf, &(pTbCfg->stbCfg.nTagCols)); pTbCfg->stbCfg.pTagSchema = (SSchema *)malloc(sizeof(SSchema) * pTbCfg->stbCfg.nTagCols); for (uint32_t i = 0; i < pTbCfg->stbCfg.nTagCols; i++) { - buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].colId); - buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pSchema[i].bytes); - buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pSchema[i].name); + buf = taosDecodeFixedI8(buf, &(pTbCfg->stbCfg.pTagSchema[i].type)); + buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].colId); + buf = taosDecodeFixedI32(buf, &pTbCfg->stbCfg.pTagSchema[i].bytes); + buf = taosDecodeStringTo(buf, pTbCfg->stbCfg.pTagSchema[i].name); } } else if (pTbCfg->type == META_CHILD_TABLE) { buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); @@ -431,79 +431,162 @@ static void metaClearTbCfg(STbCfg *pTbCfg) { } /* ------------------------ FOR QUERY ------------------------ */ -int metaGetTableInfo(SMeta *pMeta, char *tbname, STableMetaMsg **ppMsg) { - DBT key = {0}; - DBT value = {0}; - SMetaDB * pMetaDB = pMeta->pDB; - int ret; - STbCfg tbCfg; - SSchemaKey schemaKey; - DBT key1 = {0}; - DBT value1 = {0}; - uint32_t ncols; - void * pBuf; - int tlen; - STableMetaMsg *pMsg; +STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { + STbCfg * pTbCfg = NULL; + SMetaDB *pDB = pMeta->pDB; + DBT key = {0}; + DBT value = {0}; + int ret; - key.data = tbname; - key.size = strlen(tbname) + 1; + // Set key/value + key.data = &uid; + key.size = sizeof(uid); - ret = pMetaDB->pNameIdx->get(pMetaDB->pNameIdx, NULL, &key, &value, 0); + // Query + ret = pDB->pTbDB->get(pDB->pTbDB, NULL, &key, &value, 0); if (ret != 0) { - // TODO - return -1; + return NULL; } - metaDecodeTbInfo(value.data, &tbCfg); - - switch (tbCfg.type) { - case META_SUPER_TABLE: - schemaKey.uid = tbCfg.stbCfg.suid; - schemaKey.sver = 0; - - key1.data = &schemaKey; - key1.size = sizeof(schemaKey); - - ret = pMetaDB->pSchemaDB->get(pMetaDB->pSchemaDB, &key1, &value1, NULL, 0); - if (ret != 0) { - // TODO - return -1; - } - pBuf = value1.data; - pBuf = taosDecodeFixedU32(pBuf, &ncols); - - tlen = sizeof(STableMetaMsg) + (tbCfg.stbCfg.nTagCols + ncols) * sizeof(SSchema); - pMsg = calloc(1, tlen); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - strcpy(pMsg->tbFname, tbCfg.name); - pMsg->numOfTags = tbCfg.stbCfg.nTagCols; - pMsg->numOfColumns = ncols; - pMsg->tableType = tbCfg.type; - pMsg->sversion = 0; - pMsg->tversion = 0; - pMsg->suid = tbCfg.stbCfg.suid; - pMsg->tuid = tbCfg.stbCfg.suid; - for (size_t i = 0; i < tbCfg.stbCfg.nTagCols; i++) { - - } - - break; - case META_CHILD_TABLE: - ASSERT(0); - break; - case META_NORMAL_TABLE: - ASSERT(0); - break; - default: - ASSERT(0); - break; + // Decode + pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + if (pTbCfg == NULL) { + return NULL; } - *ppMsg = pMsg; + metaDecodeTbInfo(value.data, pTbCfg); - return 0; + return pTbCfg; +} + +STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { + STbCfg * pTbCfg = NULL; + SMetaDB *pDB = pMeta->pDB; + DBT key = {0}; + DBT pkey = {0}; + DBT pvalue = {0}; + int ret; + + // Set key/value + key.data = tbname; + key.size = strlen(tbname); + + // Query + ret = pDB->pNameIdx->pget(pDB->pNameIdx, NULL, &key, &pkey, &pvalue, 0); + if (ret != 0) { + return NULL; + } + + // Decode + *uid = *(tb_uid_t *)(pkey.data); + pTbCfg = (STbCfg *)malloc(sizeof(*pTbCfg)); + if (pTbCfg == NULL) { + return NULL; + } + + metaDecodeTbInfo(pvalue.data, pTbCfg); + + return pTbCfg; +} + +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { + uint32_t nCols; + SSchemaWrapper *pSW = NULL; + SMetaDB * pDB = pMeta->pDB; + int ret; + void * pBuf; + SSchema * pSchema; + SSchemaKey schemaKey = {uid, sver}; + DBT key = {0}; + DBT value = {0}; + + // Set key/value properties + key.data = &schemaKey; + key.size = sizeof(schemaKey); + + // Query + ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0); + if (ret != 0) { + return NULL; + } + + // Decode the schema + pBuf = value.data; + taosDecodeFixedI32(&pBuf, &nCols); + if (isinline) { + pSW = (SSchemaWrapper *)malloc(sizeof(*pSW) + sizeof(SSchema) * nCols); + if (pSW == NULL) { + return NULL; + } + pSW->pSchema = POINTER_SHIFT(pSW, sizeof(*pSW)); + } else { + pSW = (SSchemaWrapper *)malloc(sizeof(*pSW)); + if (pSW == NULL) { + return NULL; + } + + pSW->pSchema = (SSchema *)malloc(sizeof(SSchema) * nCols); + if (pSW->pSchema == NULL) { + free(pSW); + return NULL; + } + } + + for (int i = 0; i < nCols; i++) { + pSchema = pSW->pSchema + i; + taosDecodeFixedI8(&pBuf, &(pSchema->type)); + taosDecodeFixedI32(&pBuf, &(pSchema->colId)); + taosDecodeFixedI32(&pBuf, &(pSchema->bytes)); + taosDecodeStringTo(&pBuf, pSchema->name); + } + + return pSW; +} + +struct SMTbCursor { + DBC *pCur; +}; + +SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { + SMTbCursor *pTbCur = NULL; + SMetaDB * pDB = pMeta->pDB; + + pTbCur = (SMTbCursor *)calloc(1, sizeof(*pTbCur)); + if (pTbCur == NULL) { + return NULL; + } + + pDB->pTbDB->cursor(pDB->pTbDB, NULL, &(pTbCur->pCur), 0); + +#if 0 + DB_BTREE_STAT *sp; + pDB->pTbDB->stat(pDB->pTbDB, NULL, &sp, 0); + printf("**************** %ld\n", sp->bt_nkeys); +#endif + + return pTbCur; +} + +void metaCloseTbCursor(SMTbCursor *pTbCur) { + if (pTbCur) { + if (pTbCur->pCur) { + pTbCur->pCur->close(pTbCur->pCur); + } + free(pTbCur); + } +} + +char *metaTbCursorNext(SMTbCursor *pTbCur) { + DBT key = {0}; + DBT value = {0}; + STbCfg tbCfg; + void * pBuf; + + if (pTbCur->pCur->get(pTbCur->pCur, &key, &value, DB_NEXT) == 0) { + pBuf = value.data; + metaDecodeTbInfo(pBuf, &tbCfg); + return tbCfg.name; + } else { + return NULL; + } } \ No newline at end of file diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d881f226e4..b8fa779e70 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -510,7 +510,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; } - ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (NULL == ctgMgmt.pCluster) { CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER); } @@ -518,8 +518,8 @@ int32_t catalogInit(SCatalogCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle) { - if (NULL == clusterId || NULL == catalogHandle) { +int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { + if (NULL == catalogHandle) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -528,8 +528,7 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } - size_t clen = strlen(clusterId); - SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, clusterId, clen); + SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, (char*)&clusterId, sizeof(clusterId)); if (ctg && (*ctg)) { *catalogHandle = *ctg; @@ -542,8 +541,8 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { - ctgError("put cluster %s cache to hash failed", clusterId); + if (taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES)) { + ctgError("put cluster %"PRIx64" cache to hash failed", clusterId); tfree(clusterCtg); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 5bacbbd7e7..5979d3a147 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -55,7 +55,7 @@ int32_t ctgTestTagNum = 1; int32_t ctgTestSVersion = 1; int32_t ctgTestTVersion = 1; -char *ctgTestClusterId = "cluster1"; +uint64_t ctgTestClusterId = 0x1; char *ctgTestDbname = "1.db1"; char *ctgTestTablename = "table1"; char *ctgTestCTablename = "ctable1"; diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 73c79b2619..072ec93e4e 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -142,7 +142,8 @@ uint64_t fstStateInputLen(FstState* state); // end_addr uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data); uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes); -uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, + uint64_t nTrans); // input uint8_t fstStateInput(FstState* state, FstNode* node); uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i); @@ -255,9 +256,10 @@ typedef struct FstMeta { } FstMeta; typedef struct Fst { - FstMeta* meta; - FstSlice* data; // - FstNode* root; // + FstMeta* meta; + FstSlice* data; // + FstNode* root; // + pthread_mutex_t mtx; } Fst; // refactor simple function @@ -310,7 +312,8 @@ StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* sta void swsResultDestroy(StreamWithStateResult* result); typedef void* (*StreamCallback)(void*); -StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max); +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, + FstBoundWithData* max); void streamWithStateDestroy(StreamWithState* sws); diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ba0bd52e10..1504a69a08 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -37,6 +37,7 @@ typedef struct WriterCtx { struct { int fd; bool readOnly; + char buf[256]; } file; struct { int32_t capa; @@ -53,7 +54,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off static int writeCtxDoFlush(WriterCtx* ctx); WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); -void writerCtxDestroy(WriterCtx* w); +void writerCtxDestroy(WriterCtx* w, bool remove); typedef uint32_t CheckSummer; diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index f97a3126c8..4928e01a63 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -77,6 +77,7 @@ typedef struct TFileReader { Fst* fst; WriterCtx* ctx; TFileHeader header; + bool remove; } TFileReader; typedef struct IndexTFile { @@ -107,6 +108,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName); +TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName); TFileReader* tfileReaderCreate(WriterCtx* ctx); void tfileReaderDestroy(TFileReader* reader); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); @@ -114,10 +116,10 @@ void tfileReaderRef(TFileReader* reader); void tfileReaderUnRef(TFileReader* reader); TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t type); -void tfileWriteClose(TFileWriter* tw); +void tfileWriterClose(TFileWriter* tw); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); void tfileWriterDestroy(TFileWriter* tw); -int tfileWriterPut(TFileWriter* tw, void* data); +int tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b78c4ff258..5167196031 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -25,7 +25,7 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 4 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; @@ -52,18 +52,17 @@ typedef struct SIdxColInfo { static pthread_once_t isInit = PTHREAD_ONCE_INIT; // static void indexInit(); - static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); static void indexInterResultsDestroy(SArray* results); static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); +static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { // pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { - return -1; - } + if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -95,18 +94,16 @@ void indexClose(SIndex* sIdx) { #endif #ifdef USE_INVERTED_INDEX - indexCacheDestroy(sIdx->cache); void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { - indexCacheUnRef(*pCache); - } + if (*pCache) { indexCacheUnRef(*pCache); } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); #endif + free(sIdx->path); free(sIdx); return; } @@ -147,9 +144,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { - return ret; - } + if (ret != 0) { return ret; } } #endif @@ -179,9 +174,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } + for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -238,9 +231,7 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { - return NULL; - } + if (p == NULL) { return NULL; } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -262,9 +253,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { - return NULL; - } + if (t == NULL) { return NULL; } t->suid = suid; t->operType = oper; @@ -317,7 +306,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex - STermValueType s; + STermValueType s = kTypeValue; if (0 == indexCacheSearch(cache, query, *result, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by other opera", term->colName); @@ -336,9 +325,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { - return; - } + if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -368,23 +355,43 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType return 0; } -int indexFlushCacheTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { - return -1; +static void indexMergeSameKey(SArray* result, TFileValue* tv) { + int32_t sz = result ? taosArrayGetSize(result) : 0; + if (sz > 0) { + // TODO(yihao): remove duplicate tableid + TFileValue* lv = taosArrayGetP(result, sz - 1); + if (strcmp(lv->colVal, tv->colVal) == 0) { + taosArrayAddAll(lv->tableId, tv->tableId); + tfileValueDestroy(tv); + } else { + taosArrayPush(result, &tv); + } + } else { + taosArrayPush(result, &tv); } +} +static void indexDestroyTempResult(SArray* result) { + int32_t sz = result ? taosArrayGetSize(result) : 0; + for (size_t i = 0; i < sz; i++) { + TFileValue* tv = taosArrayGetP(result, i); + tfileValueDestroy(tv); + } + taosArrayDestroy(result); +} +int indexFlushCacheTFile(SIndex* sIdx, void* cache) { + if (sIdx == NULL) { return -1; } indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); - // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); SArray* result = taosArrayInit(1024, sizeof(void*)); - bool cn = cacheIter->next(cacheIter); - bool tn = tfileIter->next(tfileIter); + bool cn = cacheIter ? cacheIter->next(cacheIter) : false; + bool tn = tfileIter ? tfileIter->next(tfileIter) : false; while (cn == true && tn == true) { IterateValue* cv = cacheIter->getValue(cacheIter); IterateValue* tv = tfileIter->getValue(tfileIter); @@ -395,7 +402,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); taosArrayAddAll(tfv->tableId, tv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); @@ -403,13 +410,15 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { } else if (comp < 0) { TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); - taosArrayPush(result, &tfv); + + indexMergeSameKey(result, tfv); // copy to final Result; cn = cacheIter->next(cacheIter); } else { TFileValue* tfv = tfileValueCreate(tv->colVal); - taosArrayPush(result, &tfv); taosArrayAddAll(tfv->tableId, tv->val); + + indexMergeSameKey(result, tfv); // copy to final result tn = tfileIter->next(tfileIter); } @@ -418,33 +427,24 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { IterateValue* cv = cacheIter->getValue(cacheIter); TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); cn = cacheIter->next(cacheIter); } while (tn == true) { IterateValue* tv = tfileIter->getValue(tfileIter); TFileValue* tfv = tfileValueCreate(tv->colVal); + if (tv->val == NULL) { + // HO + printf("NO...."); + } taosArrayAddAll(tfv->tableId, tv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); tn = tfileIter->next(tfileIter); } - - int32_t version = CACHE_VERSION(pCache); - uint8_t colType = pCache->type; - - TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, pCache->colName, colType); - if (tw == NULL) { - indexError("faile to open file to write"); - } else { - int ret = tfileWriterPut(tw, result); - if (ret != 0) { - indexError("faile to write into tindex "); - } - } - // not free later, just put int table cache + int ret = indexGenTFile(sIdx, pCache, result); + indexDestroyTempResult(result); indexCacheDestroyImm(pCache); - tfileWriteClose(tw); indexCacheIteratorDestroy(cacheIter); tfileIteratorDestroy(tfileIter); @@ -455,9 +455,45 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { void iterateValueDestroy(IterateValue* value, bool destroy) { if (destroy) { taosArrayDestroy(value->val); + value->val = NULL; } else { - taosArrayClear(value->val); + if (value->val != NULL) { taosArrayClear(value->val); } } - free(value->colVal); + // free(value->colVal); value->colVal = NULL; } +static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { + int32_t version = CACHE_VERSION(cache); + uint8_t colType = cache->type; + + TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, cache->colName, colType); + if (tw == NULL) { + indexError("failed to open file to write"); + return -1; + } + + int ret = tfileWriterPut(tw, batch, true); + if (ret != 0) { + indexError("failed to write into tindex "); + goto END; + } + tfileWriterClose(tw); + + TFileReader* reader = tfileReaderOpen(sIdx->path, sIdx->suid, version, cache->colName); + + char buf[128] = {0}; + TFileHeader* header = &reader->header; + TFileCacheKey key = {.suid = header->suid, + .colName = header->colName, + .nColName = strlen(header->colName), + .colType = header->colType}; + pthread_mutex_lock(&sIdx->mtx); + + IndexTFile* ifile = (IndexTFile*)sIdx->tindex; + tfileCachePut(ifile->cache, &key, reader); + + pthread_mutex_unlock(&sIdx->mtx); + return ret; +END: + tfileWriterClose(tw); +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 217545d23b..0e46445a00 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 1000000 +#define MEM_TERM_LIMIT 10000 * 10 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -66,19 +66,43 @@ void indexCacheDebug(IndexCache* cache) { indexMemRef(tbl); pthread_mutex_unlock(&cache->mtx); - SSkipList* slt = tbl->mem; - SSkipListIterator* iter = tSkipListCreateIter(slt); - while (tSkipListIterNext(iter)) { - SSkipListNode* node = tSkipListIterGet(iter); - CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - if (ct != NULL) { - // TODO, add more debug info - indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + { + SSkipList* slt = tbl->mem; + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } } - } - tSkipListDestroyIter(iter); + tSkipListDestroyIter(iter); - indexMemUnRef(tbl); + indexMemUnRef(tbl); + } + + { + pthread_mutex_lock(&cache->mtx); + tbl = cache->imm; + indexMemRef(tbl); + pthread_mutex_unlock(&cache->mtx); + if (tbl != NULL) { + SSkipList* slt = tbl->mem; + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } + } + tSkipListDestroyIter(iter); + } + + indexMemUnRef(tbl); + } } void indexCacheDestroySkiplist(SSkipList* slt) { @@ -87,24 +111,28 @@ void indexCacheDestroySkiplist(SSkipList* slt) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); if (ct != NULL) { + free(ct->colVal); + free(ct); } } tSkipListDestroyIter(iter); tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { + if (cache == NULL) { return; } + MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread pthread_mutex_unlock(&cache->mtx); + + indexMemUnRef(tbl); indexMemUnRef(tbl); } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { - return; - } + if (pCache == NULL) { return; } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -114,9 +142,11 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { - return NULL; - } + if (iiter == NULL) { return NULL; } + + pthread_mutex_lock(&cache->mtx); + + indexMemRef(cache->imm); MemTable* tbl = cache->imm; iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); @@ -124,12 +154,12 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; + pthread_mutex_unlock(&cache->mtx); + return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -155,6 +185,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { taosMsleep(50); pthread_mutex_lock(&cache->mtx); } else { + indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); cache->nTerm = 1; @@ -166,17 +197,13 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -205,32 +232,11 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u IndexCache* pCache = cache; return 0; } -int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { - return -1; - } - IndexCache* pCache = cache; - SIndexTerm* term = query->term; - EIndexQueryType qtype = query->qType; - - MemTable *mem = NULL, *imm = NULL; - pthread_mutex_lock(&pCache->mtx); - mem = pCache->mem; - imm = pCache->imm; - indexMemRef(mem); - indexMemRef(imm); - pthread_mutex_unlock(&pCache->mtx); - - CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (ct == NULL) { - return -1; - } - ct->colVal = calloc(1, sizeof(char) * (term->nColVal + 1)); - memcpy(ct->colVal, term->colVal, term->nColVal); - ct->version = atomic_load_32(&pCache->version); +static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { + if (mem == NULL) { return 0; } char* key = getIndexKey(ct); - // TODO handle multi situation later, and refactor + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); @@ -251,51 +257,56 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } } tSkipListDestroyIter(iter); - cacheTermDestroy(ct); - // int32_t keyLen = CACHE_KEY_LEN(term); - // char* buf = calloc(1, keyLen); - if (qtype == QUERY_TERM) { - // - } else if (qtype == QUERY_PREFIX) { - // - } else if (qtype == QUERY_SUFFIX) { - // - } else if (qtype == QUERY_REGEX) { - // + return 0; +} +int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { + if (cache == NULL) { return -1; } + IndexCache* pCache = cache; + + MemTable *mem = NULL, *imm = NULL; + pthread_mutex_lock(&pCache->mtx); + mem = pCache->mem; + imm = pCache->imm; + indexMemRef(mem); + indexMemRef(imm); + pthread_mutex_unlock(&pCache->mtx); + + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + // indexCacheDebug(pCache); + + int ret = indexQueryMem(mem, &ct, qtype, result, s); + if (ret == 0 && *s != kTypeDeletion) { + // continue search in imm + ret = indexQueryMem(imm, &ct, qtype, result, s); } + // cacheTermDestroy(ct); + indexMemUnRef(mem); indexMemUnRef(imm); - return 0; + + return ret; } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_DEC(cache); - if (ref == 0) { - indexCacheDestroy(cache); - } + if (ref == 0) { indexCacheDestroy(cache); } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -305,9 +316,7 @@ void indexMemUnRef(MemTable* tbl) { } static void cacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { - return; - } + if (ct == NULL) { return; } free(ct->colVal); free(ct); } @@ -322,9 +331,7 @@ static int32_t compareKey(const void* l, const void* r) { // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { - return rt->version - lt->version; - } + if (cmp == 0) { return rt->version - lt->version; } return cmp; } @@ -344,9 +351,7 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { - return false; - } + if (iter == NULL) { return false; } IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); @@ -356,7 +361,8 @@ static bool indexCacheIteratorNext(Iterate* itera) { CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); iv->type = ct->operaType; - iv->colVal = ct->colVal; + iv->colVal = calloc(1, strlen(ct->colVal) + 1); + memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); taosArrayPush(iv->val, &ct->uid); } diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 18024fa391..04a08dafd2 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -354,7 +354,8 @@ uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); } -uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) { +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, + uint64_t nTrans) { uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size @@ -403,8 +404,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i FstSlice* slice = &node->data; uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans - - (i * tSizes) - tSizes; + uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - + node->nTrans - (i * tSizes) - tSizes; uint8_t* data = fstSliceData(slice, NULL); return unpackDelta(data + at, tSizes, node->end); } @@ -595,7 +596,8 @@ FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); n->nTrans = nTrans; n->sizes = sz; - n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); + n->finalOutput = + fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); } return n; } @@ -875,9 +877,7 @@ void* fstBuilderInsertInner(FstBuilder* b) { // b->wrt = NULL; return b->wrt; } -void fstBuilderFinish(FstBuilder* b) { - fstBuilderInsertInner(b); -} +void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } FstSlice fstNodeAsSlice(FstNode* node) { FstSlice* slice = &node->data; @@ -894,9 +894,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { return trn; } -void fstLastTransitionDestroy(FstLastTransition* trn) { - free(trn); -} +void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; if (trn == NULL) { return; } @@ -959,9 +957,10 @@ Fst* fstCreate(FstSlice* slice) { fst->meta->checkSum = checkSum; FstSlice* s = calloc(1, sizeof(FstSlice)); - *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); fst->data = s; + pthread_mutex_init(&fst->mtx, NULL); return fst; FST_CREAT_FAILED: @@ -973,14 +972,18 @@ void fstDestroy(Fst* fst) { free(fst->meta); fstSliceDestroy(fst->data); free(fst->data); + pthread_mutex_destroy(&fst->mtx); } free(fst); } bool fstGet(Fst* fst, FstSlice* b, Output* out) { + // dec lock range + pthread_mutex_lock(&fst->mtx); FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; + uint8_t* data = fstSliceData(b, &len); SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); @@ -988,7 +991,10 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; - if (false == fstNodeFindInput(root, inp, &res)) { return false; } + if (false == fstNodeFindInput(root, inp, &res)) { + pthread_mutex_unlock(&fst->mtx); + return false; + } FstTransition trn; fstNodeGetTransitionAt(root, res, &trn); @@ -997,6 +1003,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { + pthread_mutex_unlock(&fst->mtx); return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); @@ -1007,13 +1014,13 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { fstNodeDestroy(*node); } taosArrayDestroy(nodes); - fst->root = NULL; + pthread_mutex_unlock(&fst->mtx); *out = tOut; - return true; } FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { + // refactor later return fstStreamBuilderCreate(fst, ctx); } StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { @@ -1021,24 +1028,30 @@ StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { + // refactor later return fstStreamBuilderCreate(fst, ctx); } FstNode* fstGetRoot(Fst* fst) { - if (fst->root != NULL) { return fst->root; } CompiledAddr rAddr = fstGetRootAddr(fst); - fst->root = fstGetNode(fst, rAddr); - return fst->root; + return fstGetNode(fst, rAddr); + // pthread_mutex_lock(&fst->mtx); + // if (fst->root != NULL) { + // // pthread_mutex_unlock(&fst->mtx); + // return fst->root; + //} + // CompiledAddr rAddr = fstGetRootAddr(fst); + // fst->root = fstGetNode(fst, rAddr); + //// pthread_mutex_unlock(&fst->mtx); + // return fst->root; } + FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { + // refactor later return fstNodeCreate(fst->meta->version, addr, fst->data); } -FstType fstGetType(Fst* fst) { - return fst->meta->ty; -} -CompiledAddr fstGetRootAddr(Fst* fst) { - return fst->meta->rootAddr; -} +FstType fstGetType(Fst* fst) { return fst->meta->ty; } +CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; } Output fstEmptyFinalOutput(Fst* fst, bool* null) { Output res = 0; @@ -1053,8 +1066,7 @@ Output fstEmptyFinalOutput(Fst* fst, bool* null) { } bool fstVerify(Fst* fst) { - uint32_t checkSum = fst->meta->checkSum; - int32_t len; + uint32_t len, checkSum = fst->meta->checkSum; uint8_t* data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; if (!taosCheckChecksumWhole(data, len)) { return false; } @@ -1094,15 +1106,12 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { } } -bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { - return bound->type == Excluded ? false : true; -} +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -void fstBoundDestroy(FstBoundWithData* bound) { - free(bound); -} +void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } -StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, + FstBoundWithData* max) { StreamWithState* sws = calloc(1, sizeof(StreamWithState)); if (sws == NULL) { return NULL; } @@ -1131,7 +1140,9 @@ void streamWithStateDestroy(StreamWithState* sws) { bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { AutomationCtx* aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { - if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); } + if (fstBoundWithDataIsIncluded(min)) { + sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); + } StreamState s = {.node = fstGetRoot(sws->fst), .trans = 0, .out = {.null = false, .out = 0}, @@ -1203,7 +1214,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { uint64_t trans = s->trans; FstTransition trn; fstNodeGetTransitionAt(n, trans - 1, &trn); - StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; + StreamState s = { + .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); return true; } @@ -1260,9 +1272,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb size_t isz = taosArrayGetSize(sws->inp); uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); - for (uint32_t i = 0; i < isz; i++) { - buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); - } + for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); @@ -1327,8 +1337,8 @@ FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { } void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); - tfree(b->min); fstSliceDestroy(&b->max->data); + tfree(b->min); tfree(b->max); free(b); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 72983809d1..c6e3cee3e3 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -65,24 +65,12 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { } // iterate fst -static void* alwaysMatchStart(AutomationCtx* ctx) { - return NULL; -} -static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { - return true; -} -static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { - return true; -} -static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} -static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { - return NULL; -} -static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* alwaysMatchStart(AutomationCtx* ctx) { return NULL; } +static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { return true; } +static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { return true; } +static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } +static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } +static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { return NULL; } // prefix query, impl later static void* prefixStart(AutomationCtx* ctx) { @@ -97,9 +85,7 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val >= 0; } -static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} +static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { StartWithStateValue* ssv = (StartWithStateValue*)state; if (ssv == NULL || ctx == NULL) { return NULL; } @@ -107,7 +93,8 @@ static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { char* data = ctx->data; if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { - int val = ssv->val + 1; + int val = ssv->val + 1; + StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; @@ -118,35 +105,22 @@ static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { } return NULL; } -static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } // pattern query, impl later -static void* patternStart(AutomationCtx* ctx) { - return NULL; -} -static bool patternIsMatch(AutomationCtx* ctx, void* data) { - return true; -} -static bool patternCanMatch(AutomationCtx* ctx, void* data) { - return true; -} -static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} +static void* patternStart(AutomationCtx* ctx) { return NULL; } +static bool patternIsMatch(AutomationCtx* ctx, void* data) { return true; } +static bool patternCanMatch(AutomationCtx* ctx, void* data) { return true; } +static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } -static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { - return NULL; -} +static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } -static void* patternAcceptEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* patternAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } AutomationFunc automFuncs[] = { - {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, alwaysMatchAccpetEof}, + {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, + alwaysMatchAccpetEof}, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof} // add more search type @@ -170,11 +144,14 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { // add more search type } - char* src = (char*)data; - size_t len = strlen(src); - char* dst = (char*)malloc(len * sizeof(char) + 1); - memcpy(dst, src, len); - dst[len] = 0; + char* dst = NULL; + if (data != NULL) { + char* src = (char*)data; + size_t len = strlen(src); + dst = (char*)malloc(len * sizeof(char) + 1); + memcpy(dst, src, len); + dst[len] = 0; + } ctx->data = dst; ctx->type = atype; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index c8fbdd7690..7906dfea11 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -42,8 +42,8 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - tfLseek(ctx->file.fd, offset, 0); - nRead = tfRead(ctx->file.fd, buf, len); + // tfLseek(ctx->file.fd, offset, 0); + nRead = tfPread(ctx->file.fd, buf, len, offset); } else { // refactor later assert(0); @@ -52,7 +52,8 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // tfFsync(ctx->fd); + // taosFsyncFile(ctx->file.fd); + tfFsync(ctx->file.fd); // tfFlush(ctx->file.fd); } else { // do nothing @@ -69,12 +70,15 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { + // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); ctx->file.fd = tfOpenCreateWriteAppend(path); } else { - ctx->file.fd = tfOpenReadWrite(path); + // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.fd = tfOpenRead(path); } + memcpy(ctx->file.buf, path, strlen(path)); if (ctx->file.fd < 0) { - indexError("open file error %d", errno); + indexError("failed to open file, error %d", errno); goto END; } } else if (ctx->type == TMemory) { @@ -95,11 +99,12 @@ END: free(ctx); return NULL; } -void writerCtxDestroy(WriterCtx* ctx) { +void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { tfClose(ctx->file.fd); + if (remove) { unlink(ctx->file.buf); } } free(ctx); } @@ -138,9 +143,8 @@ int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { - return 0; -} +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } + int fstCountingWriterFlush(FstCountingWriter* write) { WriterCtx* ctx = write->wrt; ctx->flush(ctx); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 271528a437..fc4f8593a1 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -32,6 +32,7 @@ typedef struct TFileFstIter { #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) +static int tfileUidCompare(const void* a, const void* b); static int tfileStrCompare(const void* a, const void* b); static int tfileValueCompare(const void* a, const void* b, const void* param); static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); @@ -54,9 +55,7 @@ static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { - return NULL; - } + if (tcache == NULL) { return NULL; } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; @@ -75,23 +74,23 @@ TFileCache* tfileCacheCreate(const char* path) { continue; } - WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 1024 * 64); if (wc == NULL) { indexError("failed to open index:%s", file); goto End; } - TFileReader* reader = tfileReaderCreate(wc); - tfileReaderRef(reader); - // loader fst and validate it + char buf[128] = {0}; + TFileReader* reader = tfileReaderCreate(wc); TFileHeader* header = &reader->header; TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; - - char buf[128] = {0}; tfileSerialCacheKey(&key, buf); + + tfileReaderRef(reader); + // indexTable taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } taosArrayDestroyEx(files, tfileDestroyFileName); @@ -102,9 +101,7 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { - return; - } + if (tcache == NULL) { return; } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -125,9 +122,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { tfileSerialCacheKey(key, buf); TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } tfileReaderRef(*reader); return *reader; @@ -137,9 +132,10 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) tfileSerialCacheKey(key, buf); // remove last version index reader TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); - if (*p != NULL) { + if (p != NULL) { TFileReader* oldReader = *p; taosHashRemove(tcache->tableCache, buf, strlen(buf)); + oldReader->remove = true; tfileReaderUnRef(oldReader); } @@ -149,13 +145,10 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } // T_REF_INC(reader); reader->ctx = ctx; - if (0 != tfileReaderLoadHeader(reader)) { tfileReaderDestroy(reader); indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, @@ -172,12 +165,10 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { - return; - } + if (reader == NULL) { return; } // T_REF_INC(reader); fstDestroy(reader->fst); - writerCtxDestroy(reader->ctx); + writerCtxDestroy(reader->ctx, reader->remove); free(reader); } @@ -216,7 +207,8 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c char fullname[256] = {0}; snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); - WriterCtx* wcx = writerCtxCreate(TFile, fullname, true, 1024 * 1024); + WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); + if (wcx == NULL) { return NULL; } TFileHeader tfh = {0}; tfh.suid = suid; @@ -226,6 +218,21 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c return tfileWriterCreate(wcx, &tfh); } +TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName) { + char filename[128] = {0}; + int32_t coldId = 1; + tfileGenFileName(filename, suid, coldId, version); + + char fullname[256] = {0}; + snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); + WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); + if (wc == NULL) { return NULL; } + + TFileReader* reader = tfileReaderCreate(wc); + return reader; + + // tfileSerialCacheKey(&key, buf); +} TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { // char pathBuf[128] = {0}; // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); @@ -249,28 +256,29 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { return tw; } -int tfileWriterPut(TFileWriter* tw, void* data) { +int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // sort by coltype and write to tindex - __compar_fn_t fn; - - int8_t colType = tw->header.colType; - if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { - fn = tfileStrCompare; - } else { - fn = getComparFunc(colType, 0); + if (order == false) { + __compar_fn_t fn; + int8_t colType = tw->header.colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + fn = tfileStrCompare; + } else { + fn = getComparFunc(colType, 0); + } + taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); } - taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); - int32_t bufLimit = 4096, offset = 0; - char* buf = calloc(1, sizeof(char) * bufLimit); - char* p = buf; + int32_t bufLimit = 64 * 4096, offset = 0; + // char* buf = calloc(1, sizeof(char) * bufLimit); + // char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; // ugly code, refactor later for (size_t i = 0; i < sz; i++) { TFileValue* v = taosArrayGetP((SArray*)data, i); - + // taosArrayRemoveDuplicate(v->tablId, tfileUidCompare, NULL); int32_t tbsz = taosArrayGetSize(v->tableId); fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); } @@ -282,86 +290,90 @@ int tfileWriterPut(TFileWriter* tw, void* data) { int32_t tbsz = taosArrayGetSize(v->tableId); // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - if (offset + ttsz > bufLimit) { - // batch write - tw->ctx->write(tw->ctx, buf, offset); - offset = 0; - memset(buf, 0, bufLimit); - p = buf; - } + // if (offset + ttsz >= bufLimit) { + // // batch write + // indexInfo("offset: %d, ttsz: %d", offset, ttsz); + // // std::cout << "offset: " << offset << std::endl; + // // std::cout << "ttsz:" << ttsz < < < std::endl; + // tw->ctx->write(tw->ctx, buf, offset); + // offset = 0; + // memset(buf, 0, bufLimit); + // p = buf; + //} + // if (ttsz >= bufLimit) { + //} + char* buf = calloc(1, ttsz * sizeof(char)); + char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); - offset += ttsz; - p = buf + offset; + tw->ctx->write(tw->ctx, buf, ttsz); + // offset += ttsz; + // p = buf + offset; // set up value offset v->offset = tw->offset; tw->offset += ttsz; + free(buf); } - if (offset != 0) { - // write reversed data in buf to tindex - tw->ctx->write(tw->ctx, buf, offset); - } - tfree(buf); + // if (offset != 0) { + // write reversed data in buf to tindex + // tw->ctx->write(tw->ctx, buf, offset); + //} + // tfree(buf); tw->fb = fstBuilderCreate(tw->ctx, 0); if (tw->fb == NULL) { - tfileWriterDestroy(tw); + tfileWriterClose(tw); return -1; } // write fst + indexError("--------Begin----------------"); for (size_t i = 0; i < sz; i++) { // TODO, fst batch write later TFileValue* v = taosArrayGetP((SArray*)data, i); if (tfileWriteData(tw, v) == 0) { // } + indexError("data: %s, offset: %d len: %d", v->colVal, v->offset, (int)taosArrayGetSize(v->tableId)); } + indexError("--------End----------------"); fstBuilderFinish(tw->fb); fstBuilderDestroy(tw->fb); tw->fb = NULL; return 0; } -void tfileWriteClose(TFileWriter* tw) { - if (tw == NULL) { - return; - } - writerCtxDestroy(tw->ctx); +void tfileWriterClose(TFileWriter* tw) { + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { - return; - } - - writerCtxDestroy(tw->ctx); + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); - if (tfile == NULL) { - return NULL; - } + if (tfile == NULL) { return NULL; } tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile* tfile) { free(tfile); } +void IndexTFileDestroy(IndexTFile* tfile) { + tfileCacheDestroy(tfile->cache); + free(tfile); +} int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { - return ret; - } + if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; TFileCacheKey key = { .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { - return 0; - } + if (reader == NULL) { return 0; } return tfileReaderSearch(reader, query, result); } @@ -381,9 +393,7 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { - return false; - } + if (rt == NULL) { return false; } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -391,12 +401,9 @@ static bool tfileIteratorNext(Iterate* iiter) { memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); - swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { - return false; - } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; @@ -407,9 +414,8 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(Iterate)); - if (tIter == NULL) { - return NULL; - } + if (tIter == NULL) { return NULL; } + tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); tIter->st = streamBuilderIntoStream(tIter->fb); @@ -418,21 +424,22 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - Iterate* iter = calloc(1, sizeof(Iterate)); + if (reader == NULL) { return NULL; } + Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { + free(iter); return NULL; } - iter->next = tfileIteratorNext; iter->getValue = tifileIterateGetValue; + iter->val.val = taosArrayInit(1, sizeof(uint64_t)); return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } + IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -445,18 +452,19 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { - if (tf == NULL) { - return NULL; - } + if (tf == NULL) { return NULL; } TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } +static int tfileUidCompare(const void* a, const void* b) { + uint64_t l = *(uint64_t*)a; + uint64_t r = *(uint64_t*)b; + return l - r; +} static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } @@ -471,17 +479,13 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { - return NULL; - } - + if (tf == NULL) { return NULL; } + tf->colVal = val; tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { - return -1; - } + if (tf == NULL) { return -1; } taosArrayPush(tf->tableId, &val); return 0; } @@ -501,9 +505,7 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { - return -1; - } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } tw->offset += sizeof(fstOffset); return 0; } @@ -514,9 +516,7 @@ static int tfileWriteHeader(TFileWriter* writer) { memcpy(buf, (char*)header, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { - return -1; - } + if (sizeof(buf) != nwrite) { return -1; } writer->offset = nwrite; return 0; } @@ -541,18 +541,22 @@ static int tfileReaderLoadHeader(TFileReader* reader) { char buf[TFILE_HEADER_SIZE] = {0}; int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); - assert(nread == sizeof(buf)); + if (nread == -1) { + // + indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.fd, reader->ctx->file.buf); + } + // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); + return 0; } static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later - static int FST_MAX_SIZE = 16 * 1024; + static int FST_MAX_SIZE = 64 * 1024; char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); @@ -575,36 +579,33 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* int32_t total = sizeof(uint64_t) * nid; char* buf = calloc(1, total); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } - nread = ctx->read(ctx, buf, total); + nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); assert(total == nread); - for (int32_t i = 0; i < nid; i++) { - taosArrayPush(result, (uint64_t*)buf + i); - } + for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } free(buf); return 0; } void tfileReaderRef(TFileReader* reader) { + if (reader == NULL) { return; } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { + if (reader == NULL) { return; } int ref = T_REF_DEC(reader); if (ref == 0) { + // do nothing tfileReaderDestroy(reader); } } static int tfileGetFileList(const char* path, SArray* result) { DIR* dir = opendir(path); - if (NULL == dir) { - return -1; - } + if (NULL == dir) { return -1; } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { @@ -632,9 +633,7 @@ static int tfileCompare(const void* a, const void* b) { size_t bLen = strlen(bName); int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } // tfile name suid-colId-version.tindex diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 85bb5e2b15..da974ce6c4 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include "index.h" #include "indexInt.h" @@ -12,9 +13,7 @@ #include "tskiplist.h" #include "tutil.h" -void* callback(void* s) { - return s; -} +void* callback(void* s) { return s; } static std::string fileName = "/tmp/tindex.tindex"; class FstWriter { @@ -34,7 +33,7 @@ class FstWriter { fstBuilderFinish(_b); fstBuilderDestroy(_b); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: @@ -44,7 +43,8 @@ class FstWriter { class FstReadMemory { public: - FstReadMemory(size_t size) { + FstReadMemory(size_t size, const std::string& fileName = fileName) { + tfInit(); _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; @@ -102,7 +102,8 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); + tfCleanup(); } private: @@ -167,8 +168,44 @@ void checkFstCheckIterator() { delete m; tfCleanup(); } -int main() { - checkFstCheckIterator(); + +void fst_get(Fst* fst) { + for (int i = 0; i < 10000; i++) { + std::string term = "Hello"; + FstSlice key = fstSliceCreate((uint8_t*)term.c_str(), term.size()); + uint64_t offset = 0; + bool ret = fstGet(fst, &key, &offset); + if (ret == false) { + std::cout << "not found" << std::endl; + } else { + std::cout << "found value:" << offset << std::endl; + } + } +} + +#define NUM_OF_THREAD 10 +void validateTFile(char* arg) { + tfInit(); + + std::thread threads[NUM_OF_THREAD]; + // std::vector threads; + TFileReader* reader = tfileReaderOpen(arg, 0, 295868, "tag1"); + + for (int i = 0; i < NUM_OF_THREAD; i++) { + threads[i] = std::thread(fst_get, reader->fst); + // threads.push_back(fst_get, reader->fst); + // std::thread t(fst_get, reader->fst); + } + for (int i = 0; i < NUM_OF_THREAD; i++) { + // wait join + threads[i].join(); + } + tfCleanup(); +} +int main(int argc, char* argv[]) { + if (argc > 1) { validateTFile(argv[1]); } + // checkFstCheckIterator(); // checkFstPrefixSearch(); + return 1; } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index b3e385192f..080becccf1 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include "index.h" #include "indexInt.h" #include "index_cache.h" @@ -25,6 +26,9 @@ #include "tskiplist.h" #include "tutil.h" using namespace std; + +#define NUM_OF_THREAD 10 + class DebugInfo { public: DebugInfo(const char* str) : info(str) { @@ -41,6 +45,7 @@ class DebugInfo { private: std::string info; }; + class FstWriter { public: FstWriter() { @@ -57,7 +62,7 @@ class FstWriter { fstBuilderFinish(_b); fstBuilderDestroy(_b); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: @@ -76,9 +81,7 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { - return false; - } + if (nRead <= 0) { return false; } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -104,9 +107,7 @@ class FstReadMemory { StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; - while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { - result.push_back((uint64_t)(rt->out.out)); - } + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { result.push_back((uint64_t)(rt->out.out)); } return true; } bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { @@ -120,7 +121,7 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, true); } private: @@ -182,9 +183,7 @@ void checkFstPerf() { delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); - if (m->init()) { - printf("success to init fst read"); - } + if (m->init()) { printf("success to init fst read"); } Performance_fstReadRecords(m); delete m; } @@ -338,6 +337,8 @@ class TFileObj { TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) { colId_ = 10; + reader_ = NULL; + writer_ = NULL; // Do Nothing // } @@ -346,10 +347,8 @@ class TFileObj { tfileReaderDestroy(reader_); reader_ = NULL; } - if (writer_ == NULL) { - InitWriter(); - } - return tfileWriterPut(writer_, tv); + if (writer_ == NULL) { InitWriter(); } + return tfileWriterPut(writer_, tv, false); } bool InitWriter() { TFileHeader header; @@ -388,12 +387,8 @@ class TFileObj { return tfileReaderSearch(reader_, query, result); } ~TFileObj() { - if (writer_) { - tfileWriterDestroy(writer_); - } - if (reader_) { - tfileReaderDestroy(reader_); - } + if (writer_) { tfileWriterDestroy(writer_); } + if (reader_) { tfileReaderDestroy(reader_); } } private: @@ -439,7 +434,7 @@ static TFileValue* genTFileValue(const char* val) { memcpy(tv->colVal, val, vlen); tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); - for (size_t i = 0; i < 10; i++) { + for (size_t i = 0; i < 200; i++) { uint64_t v = i; taosArrayPush(tv->tableId, &v); } @@ -452,22 +447,17 @@ static void destroyTFileValue(void* val) { free(tv); } TEST_F(IndexTFileEnv, test_tfile_write) { - TFileValue* v1 = genTFileValue("c"); - TFileValue* v2 = genTFileValue("ab"); - TFileValue* v3 = genTFileValue("b"); - TFileValue* v4 = genTFileValue("d"); + TFileValue* v1 = genTFileValue("ab"); SArray* data = (SArray*)taosArrayInit(4, sizeof(void*)); taosArrayPush(data, &v1); - taosArrayPush(data, &v2); - taosArrayPush(data, &v3); - taosArrayPush(data, &v4); + // taosArrayPush(data, &v2); + // taosArrayPush(data, &v3); + // taosArrayPush(data, &v4); fObj->Put(data); - for (size_t i = 0; i < taosArrayGetSize(data); i++) { - destroyTFileValue(taosArrayGetP(data, i)); - } + for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); } taosArrayDestroy(data); std::string colName("voltage"); @@ -478,7 +468,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); - assert(taosArrayGetSize(result) == 10); + assert(taosArrayGetSize(result) == 200); indexTermDestroy(term); // tfileWriterDestroy(twrite); @@ -544,6 +534,7 @@ TEST_F(IndexCacheEnv, cache_test) { SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); + // indexTermDestry(term); } { std::string colVal("v3"); @@ -569,7 +560,8 @@ TEST_F(IndexCacheEnv, cache_test) { colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); } - + coj->Debug(); + std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), @@ -582,6 +574,8 @@ TEST_F(IndexCacheEnv, cache_test) { colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); } + coj->Debug(); + std::cout << "--------second----------" << std::endl; { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { @@ -602,7 +596,7 @@ TEST_F(IndexCacheEnv, cache_test) { STermValueType valType; coj->Get(&query, colId, 10000, ret, &valType); - // std::cout << "size : " << taosArrayGetSize(ret) << std::endl; + std::cout << "size : " << taosArrayGetSize(ret) << std::endl; assert(taosArrayGetSize(ret) == 4); } { @@ -635,6 +629,37 @@ class IndexObj { } return ret; } + int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", + size_t numOfTable = 100 * 10000) { + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < numOfTable; i++) { + int ret = Put(terms, i); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + return numOfTable; + } + int WriteMultiMillonData(const std::string& colName, const std::string& colVal = "Hello world", + size_t numOfTable = 100 * 10000) { + std::string tColVal = colVal; + for (int i = 0; i < numOfTable; i++) { + tColVal[tColVal.size() - 1] = 'a' + i % 26; + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < 10; i++) { + int ret = Put(terms, i); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + return numOfTable; + } + int Put(SIndexMultiTerm* fvs, uint64_t uid) { numOfWrite += taosArrayGetSize(fvs); return indexPut(idx, fvs, uid); @@ -645,14 +670,33 @@ class IndexObj { return indexSearch(idx, multiQ, result); } + int SearchOne(const std::string& colName, const std::string& colVal) { + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + indexMultiTermQueryAdd(mq, term, QUERY_TERM); + + SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); + if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; } + return taosArrayGetSize(result); + // assert(taosArrayGetSize(result) == targetSize); + } + void PutOne(const std::string& colName, const std::string& colVal) { + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + Put(terms, 10); + indexMultiTermDestroy(terms); + } void Debug() { std::cout << "numOfWrite:" << numOfWrite << std::endl; std::cout << "numOfRead:" << numOfRead << std::endl; } ~IndexObj() { - indexClose(idx); indexCleanUp(); + indexClose(idx); } private: @@ -665,25 +709,26 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: virtual void SetUp() { + tfInit(); index = new IndexObj(); // } virtual void TearDown() { - // r delete index; + tfCleanup(); } IndexObj* index; }; TEST_F(IndexEnv2, testIndexOpen) { - std::string path = "/tmp"; + std::string path = "/tmp/test"; if (index->Init(path) != 0) { std::cout << "failed to init index" << std::endl; exit(1); } - int targetSize = 100; + int targetSize = 200; { - std::string colName("tag1"), colVal("Hello world"); + std::string colName("tag1"), colVal("Hello"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -697,8 +742,8 @@ TEST_F(IndexEnv2, testIndexOpen) { indexMultiTermDestroy(terms); } { - size_t size = 100; - std::string colName("tag1"), colVal("hello world"); + size_t size = 200; + std::string colName("tag1"), colVal("hello"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -711,10 +756,24 @@ TEST_F(IndexEnv2, testIndexOpen) { } indexMultiTermDestroy(terms); } + { + size_t size = 200; + std::string colName("tag1"), colVal("Hello"); + + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = size * 3; i < size * 4; i++) { + int tableId = i; + int ret = index->Put(terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } { - std::string colName("tag1"), colVal("Hello world"); - + std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -722,27 +781,65 @@ TEST_F(IndexEnv2, testIndexOpen) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); index->Search(mq, result); - assert(taosArrayGetSize(result) == targetSize); - } -} -TEST_F(IndexEnv2, testIndex_CachePut) { - std::string path = "/tmp"; - if (index->Init(path) != 0) { + std::cout << "target size: " << taosArrayGetSize(result) << std::endl; + assert(taosArrayGetSize(result) == 400); } } -TEST_F(IndexEnv2, testIndexr_TFilePut) { - std::string path = "/tmp"; +TEST_F(IndexEnv2, testIndex_TrigeFlush) { + std::string path = "/tmp/test"; if (index->Init(path) != 0) { + // r + std::cout << "failed to init" << std::endl; + } + int numOfTable = 100 * 10000; + index->WriteMillonData("tag1", "Hello", numOfTable); + int target = index->SearchOne("tag1", "Hello"); + assert(numOfTable == target); +} + +static void write_and_search(IndexObj* idx) { + std::string colName("tag1"), colVal("Hello"); + + int target = idx->SearchOne("tag1", "Hello"); + idx->PutOne(colName, colVal); +} +TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { + std::string path = "/tmp/cache_and_tfile"; + if (index->Init(path) != 0) { + // opt + } + index->WriteMultiMillonData("tag1", "Hello", 200000); + std::thread threads[NUM_OF_THREAD]; + + for (int i = 0; i < NUM_OF_THREAD; i++) { + // + threads[i] = std::thread(write_and_search, index); + } + for (int i = 0; i < NUM_OF_THREAD; i++) { + // TOD + threads[i].join(); } } -TEST_F(IndexEnv2, testIndex_CacheSearch) { +TEST_F(IndexEnv2, testIndex_multi_thread_write) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } -TEST_F(IndexEnv2, testIndex_TFileSearch) { +TEST_F(IndexEnv2, testIndex_multi_thread_read) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} +} + +TEST_F(IndexEnv2, testIndex_restart) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} +} + +TEST_F(IndexEnv2, testIndex_performance) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} +} +TEST_F(IndexEnv2, testIndexMultiTag) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} } diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 0febc5ea33..22806969af 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -125,7 +125,6 @@ typedef struct SCreatedTableInfo { SArray *pTagNames; // create by using super table, tag name SArray *pTagVals; // create by using super table, tag value char *fullname; // table full name - STagData tagdata; // true tag data, super table full name is in STagData int8_t igExist; // ignore if exists } SCreatedTableInfo; diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 186a4869e6..4bbe6ab907 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -102,7 +102,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* p * Destroy the meta data request structure. * @param pMetaInfo */ -void qParserClearupMetaRequestInfo(SCatalogReq* pMetaInfo); +void qParserCleanupMetaRequestInfo(SCatalogReq* pMetaInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index bf59a0f80a..789f5a98e2 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -188,7 +188,7 @@ cmd ::= COMPACT VNODES IN LP exprlist(Y) RP. { setCompactVnodeSql(pInfo, TSDB // And "ids" is an identifer-or-string. %type ids {SToken} ids(A) ::= ID(X). {A = X; } -ids(A) ::= STRING(X). {A = X; } +//ids(A) ::= STRING(X). {A = X; } %type ifexists {SToken} ifexists(X) ::= IF EXISTS. { X.n = 1;} diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index 3d12f0f3b7..9122b0df96 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -692,7 +692,6 @@ void freeCreateTableInfo(void* p) { taosArrayDestroy(pInfo->pTagNames); taosArrayDestroyEx(pInfo->pTagVals, freeItem); tfree(pInfo->fullname); - tfree(pInfo->tagdata.data); } SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type) { diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index bdbc095861..792c0db266 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -26,15 +26,13 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in } SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { - SCreateAcctMsg* pMsg = (SCreateAcctMsg*)calloc(1, sizeof(SCreateAcctMsg)); - if (pMsg == NULL) { - // tscError("0x%" PRIx64 " failed to malloc for query msg", id); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + SCreateAcctMsg *pCreateMsg = (SCreateAcctMsg *) calloc(1, sizeof(SCreateAcctMsg)); + if (pCreateMsg == NULL) { + qError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } - SCreateAcctMsg *pCreateMsg = (SCreateAcctMsg *) calloc(1, sizeof(SCreateAcctMsg)); - SToken *pName = &pInfo->pMiscInfo->user.user; SToken *pPwd = &pInfo->pMiscInfo->user.passwd; @@ -67,17 +65,18 @@ SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in } *outputLen = sizeof(SCreateAcctMsg); - return pMsg; + return pCreateMsg; } + SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); if (pName->n >= TSDB_USER_LEN) { return NULL; } - SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg)); if (pMsg == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return NULL; } @@ -90,7 +89,6 @@ SShowMsg* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); pShowMsg->type = pShowInfo->showType; - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { SToken* pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query @@ -339,7 +337,7 @@ SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* p SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid host name (name too long, maximum length 128)"; const char* msg2 = "dnode name can not be string"; - const char* msg3 = "port should be an integer that is less than 65535"; + const char* msg3 = "port should be an integer that is less than 65535 and greater than 0"; const char* msg4 = "failed prepare create dnode message"; if (taosArrayGetSize(pInfo->pMiscInfo->a) != 2) { @@ -363,7 +361,7 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs int64_t val = 0; toInteger(port->z, port->n, 10, &val, &isSign); - if (val >= UINT16_MAX) { + if (val >= UINT16_MAX || val <= 0) { buildInvalidOperationMsg(pMsgBuf, msg3); return NULL; } @@ -384,7 +382,6 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); - char* end = NULL; SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg)); pDrop->dnodeId = strtoll(pzName->z, &end, 10); @@ -398,5 +395,4 @@ SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf } return pDrop; -} - +} \ No newline at end of file diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 7160b13eba..fd99cb6f66 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { } static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, - SMsgBuf* pMsgBuf) { + SEpSet* pEpSet, void** pExtension, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid name"; const char* msg2 = "wildcard string should be less than %d characters"; const char* msg3 = "database name too long"; @@ -31,57 +31,87 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** ou * wildcard in like clause in pInfo->pMiscInfo->a[1] */ int16_t showType = pShowInfo->showType; - if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { - SToken* pDbPrefixToken = &pShowInfo->prefix; - if (pDbPrefixToken->type != 0) { - if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long - return buildInvalidOperationMsg(pMsgBuf, msg3); - } + if (showType == TSDB_MGMT_TABLE_TABLE) { + SVShowTablesReq* pShowReq = calloc(1, sizeof(SVShowTablesReq)); - if (pDbPrefixToken->n <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg5); - } + SArray* array = NULL; + SName name = {0}; + tNameSetDbName(&name, pCtx->acctId, pCtx->db, strlen(pCtx->db)); - if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); - // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); - // if (ret != TSDB_CODE_SUCCESS) { - // return buildInvalidOperationMsg(pMsgBuf, msg1); - // } + catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, 0, &array); + + SVgroupInfo* info = taosArrayGet(array, 0); + pShowReq->head.vgId = htonl(info->vgId); + pEpSet->numOfEps = info->numOfEps; + pEpSet->inUse = info->inUse; + + for(int32_t i = 0; i < pEpSet->numOfEps; ++i) { + strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); + pEpSet->port[i] = info->epAddr[i].port; } - // show table/stable like 'xxxx', set the like pattern for show tables - SToken* pPattern = &pShowInfo->pattern; - if (pPattern->type != 0) { - if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { - return buildInvalidOperationMsg(pMsgBuf, msg4); + *outputLen = sizeof(SVShowTablesReq); + *output = pShowReq; + + *pExtension = array; + } else { + if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { + SToken* pDbPrefixToken = &pShowInfo->prefix; + if (pDbPrefixToken->type != 0) { + if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (pDbPrefixToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); + // if (ret != TSDB_CODE_SUCCESS) { + // return buildInvalidOperationMsg(pMsgBuf, msg1); + // } } - pPattern->n = strdequote(pPattern->z); - if (pPattern->n <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg6); + // show table/stable like 'xxxx', set the like pattern for show tables + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type != 0) { + if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + pPattern->n = strdequote(pPattern->z); + if (pPattern->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + if (pPattern->n > tsMaxWildCardsLen) { + char tmp[64] = {0}; + sprintf(tmp, msg2, tsMaxWildCardsLen); + return buildInvalidOperationMsg(pMsgBuf, tmp); + } + } + } else if (showType == TSDB_MGMT_TABLE_VNODES) { + if (pShowInfo->prefix.type == 0) { + return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); } - if (pPattern->n > tsMaxWildCardsLen) { - char tmp[64] = {0}; - sprintf(tmp, msg2, tsMaxWildCardsLen); - return buildInvalidOperationMsg(pMsgBuf, tmp); + if (pShowInfo->prefix.type == TK_STRING) { + pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); } } - } else if (showType == TSDB_MGMT_TABLE_VNODES) { - if (pShowInfo->prefix.type == 0) { - return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); - } - if (pShowInfo->prefix.type == TK_STRING) { - pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); - } + *pEpSet = pCtx->mgmtEpSet; + *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len); + *outputLen = sizeof(SShowMsg) /* + htons(pShowMsg->payloadLen)*/; } - *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf->buf, pMsgBuf->len); - *outputLen = sizeof(SShowMsg) /* + htons(pShowMsg->payloadLen)*/; return TSDB_CODE_SUCCESS; } @@ -333,7 +363,6 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p // too long tag values will return invalid sql, not be truncated automatically SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); STableComInfo tinfo = getTableInfo(pSuperTableMeta); - STagData* pTag = &pCreateTableInfo->tagdata; SKVRowBuilder kvRowBuilder = {0}; if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { @@ -433,22 +462,6 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p SSchema* pSchema = &pTagSchema[i]; SToken* pItem = taosArrayGet(pValList, i); - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->n > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return buildInvalidOperationMsg(pMsgBuf, msg3); - } - } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - // if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { - //// code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision); - // if (code != TSDB_CODE_SUCCESS) { - // return buildInvalidOperationMsg(pMsgBuf, msg4); - // } - // } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { - // pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision); - // } - } - char tmpTokenBuf[TSDB_MAX_TAGS_LEN] = {0}; SKvParam param = {.builder = &kvRowBuilder, .schema = pSchema}; @@ -608,8 +621,9 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm } case TSDB_SQL_SHOW: { - code = setShowInfo(&pInfo->pMiscInfo->showOpt, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, pMsgBuf); - pDcl->msgType = TDMT_MND_SHOW; + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; + code = setShowInfo(pShowInfo, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet, &pDcl->pExtension, pMsgBuf); + pDcl->msgType = (pShowInfo->showType == TSDB_MGMT_TABLE_TABLE)? TDMT_VND_SHOW_TABLES:TDMT_MND_SHOW; break; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 991bde5ed2..66966f75db 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -64,64 +64,6 @@ typedef struct SInsertParseContext { SInsertStmtInfo* pOutput; } SInsertParseContext; -static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) { - errno = 0; - *value = strtold(pToken->z, endPtr); - - // not a valid integer number, return error - if ((*endPtr - pToken->z) != pToken->n) { - return TK_ILLEGAL; - } - - return pToken->type; -} - -static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { - errno = 0; - int32_t ret = 0; - - char* endPtr = NULL; - if (type == TK_FLOAT) { - double v = strtod(z, &endPtr); - if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { - ret = -1; - } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { - ret = -1; - } else { - *value = (int64_t) round(v); - } - - errno = 0; - return ret; - } - - int32_t radix = 10; - if (type == TK_HEX) { - radix = 16; - } else if (type == TK_BIN) { - radix = 2; - } - - // the string may be overflow according to errno - if (!issigned) { - const char *p = z; - while(*p != 0 && *p == ' ') p++; - if (*p != 0 && *p == '-') { return -1;} - - *value = strtoull(z, &endPtr, radix); - } else { - *value = strtoll(z, &endPtr, radix); - } - - // not a valid integer number, return error - if (endPtr - z != n || errno == ERANGE) { - ret = -1; - } - - errno = 0; - return ret; -} - static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -159,10 +101,8 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char tableName[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&name, tableName); - SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx; CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); - SVgroupInfo vg; CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); @@ -349,207 +289,6 @@ static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *p return TSDB_CODE_SUCCESS; } -//static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { -// if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && -// type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || -// (pToken->n == 0) || (type == TK_RP)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); -// } -// -// if (IS_NUMERIC_TYPE(type) && pToken->n == 0) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); -// } -// -// // Remove quotation marks -// if (TK_STRING == type) { -// if (pToken->n >= TSDB_MAX_BYTES_PER_ROW) { -// return buildSyntaxErrMsg(pMsgBuf, "too long string", pToken->z); -// } -// -// // delete escape character: \\, \', \" -// char delim = pToken->z[0]; -// int32_t cnt = 0; -// int32_t j = 0; -// for (uint32_t k = 1; k < pToken->n - 1; ++k) { -// if (pToken->z[k] == '\\' || (pToken->z[k] == delim && pToken->z[k + 1] == delim)) { -// tmpTokenBuf[j] = pToken->z[k + 1]; -// cnt++; -// j++; -// k++; -// continue; -// } -// tmpTokenBuf[j] = pToken->z[k]; -// j++; -// } -// -// tmpTokenBuf[j] = 0; -// pToken->z = tmpTokenBuf; -// pToken->n -= 2 + cnt; -// } -// -// return TSDB_CODE_SUCCESS; -//} - -//static FORCE_INLINE int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { -// int64_t iv; -// char *endptr = NULL; -// bool isSigned = false; -// -// CHECK_CODE(checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf)); -// -// if (isNullStr(pToken)) { -// if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { -// int64_t tmpVal = 0; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// return func(getNullValue(pSchema->type), 0, param); -// } -// -// switch (pSchema->type) { -// case TSDB_DATA_TYPE_BOOL: { -// if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) { -// if (strncmp(pToken->z, "true", pToken->n) == 0) { -// return func(&TRUE_VALUE, pSchema->bytes, param); -// } else if (strncmp(pToken->z, "false", pToken->n) == 0) { -// return func(&FALSE_VALUE, pSchema->bytes, param); -// } else { -// return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); -// } -// } else if (pToken->type == TK_INTEGER) { -// return func(((strtoll(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); -// } else if (pToken->type == TK_FLOAT) { -// return func(((strtod(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, param); -// } else { -// return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); -// } -// } -// -// case TSDB_DATA_TYPE_TINYINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); -// } else if (!IS_VALID_TINYINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); -// } -// -// uint8_t tmpVal = (uint8_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_UTINYINT:{ -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); -// } else if (!IS_VALID_UTINYINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); -// } -// uint8_t tmpVal = (uint8_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_SMALLINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); -// } else if (!IS_VALID_SMALLINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); -// } -// int16_t tmpVal = (int16_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_USMALLINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); -// } else if (!IS_VALID_USMALLINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); -// } -// uint16_t tmpVal = (uint16_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_INT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); -// } else if (!IS_VALID_INT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); -// } -// int32_t tmpVal = (int32_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_UINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); -// } else if (!IS_VALID_UINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); -// } -// uint32_t tmpVal = (uint32_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_BIGINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); -// } else if (!IS_VALID_BIGINT(iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); -// } -// return func(&iv, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_UBIGINT: { -// if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); -// } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { -// return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); -// } -// uint64_t tmpVal = (uint64_t)iv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_FLOAT: { -// double dv; -// if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); -// } -// if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); -// } -// float tmpVal = (float)dv; -// return func(&tmpVal, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_DOUBLE: { -// double dv; -// if (TK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); -// } -// if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { -// return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); -// } -// return func(&dv, pSchema->bytes, param); -// } -// -// case TSDB_DATA_TYPE_BINARY: { -// // too long values will return invalid sql, not be truncated automatically -// if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { -// return buildSyntaxErrMsg(pMsgBuf, "string data overflow", pToken->z); -// } -// return func(pToken->z, pToken->n, param); -// } -// case TSDB_DATA_TYPE_NCHAR: { -// return func(pToken->z, pToken->n, param); -// } -// case TSDB_DATA_TYPE_TIMESTAMP: { -// int64_t tmpVal; -// if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { -// return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); -// } -// return func(&tmpVal, pSchema->bytes, param); -// } -// } -// -// return TSDB_CODE_FAILED; -//} - // pSql -> tag1_name, ...) static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { int32_t nCols = pColList->numOfCols; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 2ccd76723b..710cf4b5d0 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -58,13 +58,11 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { } else { SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code. return terrno; } - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(NULL, &pCatalog); - code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, pCxt->ctx.requestId, pCxt->pMsg, pCxt->msgLen); + int32_t code = qParserValidateSqlNode(pCxt->ctx.pCatalog, &info, pQueryInfo, pCxt->ctx.requestId, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } @@ -220,7 +218,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* p return code; } -void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) { +void qParserCleanupMetaRequestInfo(SCatalogReq* pMetaReq) { if (pMetaReq == NULL) { return; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 20f330247e..22545255a3 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -563,16 +563,6 @@ TAOS_FIELD createField(const SSchema* pSchema) { return f; } -SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name) { - SSchema s = {0}; - s.type = type; - s.bytes = bytes; - s.colId = colId; - - tstrncpy(s.name, name, tListLen(s.name)); - return s; -} - void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema) { pColumn->uid = uid; pColumn->flag = flag; @@ -581,7 +571,9 @@ void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t fla pColumn->info.type = pSchema->type; if (tableName != NULL) { - snprintf(pColumn->name, tListLen(pColumn->name), "%s.%s", tableName, pSchema->name); + char n[TSDB_COL_NAME_LEN + 1 + TSDB_TABLE_NAME_LEN] = {0}; + snprintf(n, tListLen(n), "%s.%s", tableName, pSchema->name); + tstrncpy(pColumn->name, n, tListLen(pColumn->name)); } else { tstrncpy(pColumn->name, pSchema->name, tListLen(pColumn->name)); } @@ -596,7 +588,10 @@ SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSc c.info.type = pSchema->type; if (tableName != NULL) { - snprintf(c.name, tListLen(c.name), "%s.%s", tableName, pSchema->name); + char n[TSDB_COL_NAME_LEN + 1 + TSDB_TABLE_NAME_LEN] = {0}; + snprintf(n, tListLen(n), "%s.%s", tableName, pSchema->name); + + tstrncpy(c.name, n, tListLen(c.name)); } else { tstrncpy(c.name, pSchema->name, tListLen(c.name)); } @@ -1649,9 +1644,9 @@ static bool isNullStr(SToken *pToken) { } static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { - if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && - type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || - (pToken->n == 0) || (type == TK_RP)) { + if ((pToken->type != TK_NOW && pToken->type != TK_INTEGER && pToken->type != TK_STRING && pToken->type != TK_FLOAT && pToken->type != TK_BOOL && + pToken->type != TK_NULL && pToken->type != TK_HEX && pToken->type != TK_OCT && pToken->type != TK_BIN) || + (pToken->n == 0) || (pToken->type == TK_RP)) { return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } @@ -1795,7 +1790,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); } else if (!IS_VALID_TINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); @@ -1806,7 +1801,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_UTINYINT:{ - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (!IS_VALID_UTINYINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); @@ -1816,7 +1811,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); } else if (!IS_VALID_SMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); @@ -1826,7 +1821,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); } else if (!IS_VALID_USMALLINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); @@ -1836,7 +1831,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); } else if (!IS_VALID_INT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); @@ -1846,7 +1841,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); } else if (!IS_VALID_UINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); @@ -1856,7 +1851,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); } else if (!IS_VALID_BIGINT(iv)) { return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); @@ -1865,7 +1860,7 @@ int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t ti } case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, pToken->type, &iv, &isSigned)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); } else if (!IS_VALID_UBIGINT((uint64_t)iv)) { return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index a651e6c1df..3be358fec8 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -97,8 +97,8 @@ public: int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; - char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(pTableName, db); const char* tname = tNameGetTableName(pTableName); int32_t code = copyTableSchemaMeta(db, tname, &table); @@ -111,6 +111,7 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo + vgInfo->vgId = 1; return 0; } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 423aa1a7be..a67a9a8be8 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -99,7 +99,7 @@ void sqlCheck(const char* sql, bool valid) { } destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -159,7 +159,7 @@ TEST(testCase, validateAST_test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -205,7 +205,7 @@ TEST(testCase, function_Test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -251,7 +251,7 @@ TEST(testCase, function_Test2) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -296,7 +296,7 @@ TEST(testCase, function_Test3) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 6); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -342,7 +342,7 @@ TEST(testCase, function_Test4) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -393,7 +393,7 @@ TEST(testCase, function_Test5) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -507,7 +507,7 @@ TEST(testCase, function_Test6) { ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -569,7 +569,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, numOfCols); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -625,7 +625,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -648,7 +648,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(ret, 0); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } @@ -678,7 +678,7 @@ TEST(testCase, function_Test6) { ASSERT_NE(ret, 0); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); //=============================================================================================================== info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); @@ -700,7 +700,7 @@ TEST(testCase, function_Test6) { ASSERT_NE(ret, 0); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index f9b1722106..04c3a7d81a 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -102,7 +102,7 @@ void generateLogicplan(const char* sql) { printf("%s\n", str); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } } @@ -163,7 +163,7 @@ TEST(testCase, planner_test) { printf("%s\n", str); destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 23c0aae15f..3ab6a6531c 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -714,7 +714,7 @@ TEST(testCase, extractMeta_test) { ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - qParserClearupMetaRequestInfo(&req); + qParserCleanupMetaRequestInfo(&req); destroySqlInfo(&info1); } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 8388458b4c..97c9cec7c7 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -207,6 +207,7 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { } taosArrayPush(currentLevel, &subplan); pCxt->pCurrentSubplan = subplan; + ++(pCxt->pDag->numOfSubplans); return subplan; } @@ -293,11 +294,14 @@ static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SArray* vgs = (SArray*)pPlanNode->pExtInfo; size_t numOfVg = taosArrayGetSize(vgs); for (int32_t i = 0; i < numOfVg; ++i) { + STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(vgs, i); vgroupInfoToEpSet(&blocks->vg, &subplan->execEpSet); subplan->pNode = NULL; subplan->pDataSink = createDataInserter(pCxt, blocks); + subplan->type = QUERY_TYPE_MODIFY; + RECOVERY_CURRENT_SUBPLAN(pCxt); } } diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 11a31d15eb..4b408e67db 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -100,6 +100,6 @@ TEST(testCase, planner_test) { // ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); // // destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); +// qParserCleanupMetaRequestInfo(&req); // destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 4296e82a56..808c1e19f9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,8 +1,9 @@ -#include "tmsg.h" -#include "query.h" #include "qworker.h" -#include "qworkerInt.h" +#include "tname.h" #include "planner.h" +#include "query.h" +#include "qworkerInt.h" +#include "tmsg.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -136,7 +137,6 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, return TSDB_CODE_SUCCESS; } - static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); @@ -155,8 +155,6 @@ static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t s return TSDB_CODE_SUCCESS; } - - static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } @@ -180,12 +178,10 @@ static int32_t qwAcquireTask(int32_t rwType, SQWSchStatus *sch, uint64_t qId, ui return qwAcquireTaskImpl(rwType, sch, qId, tId, task); } - static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { QW_UNLOCK(rwType, &sch->tasksLock); } - int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int8_t status, int32_t eOpt, SQWTaskStatus **task) { int32_t code = 0; @@ -232,7 +228,6 @@ int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t return TSDB_CODE_SUCCESS; } - static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) { SQWSchStatus *tsch = NULL; QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &tsch, QW_NOT_EXIST_ADD)); @@ -251,8 +246,6 @@ static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_ QW_RET(code); } - - static FORCE_INLINE int32_t qwAcquireTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerResCache **res) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); @@ -444,8 +437,6 @@ _return: QW_RET(code); } - - int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -479,7 +470,6 @@ int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t return TSDB_CODE_SUCCESS; } - int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -547,8 +537,6 @@ _return: QW_RET(code); } - - int32_t qwBuildAndSendQueryRsp(SRpcMsg *pMsg, int32_t code) { SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -634,7 +622,6 @@ int32_t qwBuildAndSendFetchRsp(SRpcMsg *pMsg, void *data) { return TSDB_CODE_SUCCESS; } - int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp)); pRsp->code = code; @@ -648,7 +635,6 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) { }; rpcSendResponse(&rpcRsp); - return TSDB_CODE_SUCCESS; } @@ -665,11 +651,71 @@ int32_t qwBuildAndSendDropRsp(SRpcMsg *pMsg, int32_t code) { }; rpcSendResponse(&rpcRsp); - return TSDB_CODE_SUCCESS; } +int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { + int32_t numOfCols = 6; + int32_t msgSize = sizeof(SVShowTablesRsp) + sizeof(SSchema) * numOfCols; + SVShowTablesRsp *pRsp = (SVShowTablesRsp *)rpcMallocCont(msgSize); + + int32_t cols = 0; + SSchema *pSchema = pRsp->metaInfo.pSchema; + + const SSchema *s = tGetTbnameColumnSchema(); + *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "name"); + pSchema++; + + int32_t type = TSDB_DATA_TYPE_TIMESTAMP; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "created"); + pSchema++; + + type = TSDB_DATA_TYPE_SMALLINT; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "columns"); + pSchema++; + + *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "stable"); + pSchema++; + + type = TSDB_DATA_TYPE_BIGINT; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "uid"); + pSchema++; + + type = TSDB_DATA_TYPE_INT; + *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "vgId"); + + assert(cols == numOfCols); + pRsp->metaInfo.numOfColumns = htonl(cols); + + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pRsp, + .contLen = msgSize, + .code = code, + }; + + rpcSendResponse(&rpcMsg); + return TSDB_CODE_SUCCESS; +} + +int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) { + SVShowTablesFetchRsp *pRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp)); + int32_t handle = htonl(pFetchReq->id); + + pRsp->numOfRows = 0; + SRpcMsg rpcMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .pCont = pRsp, + .contLen = sizeof(*pRsp), + .code = 0, + }; + + rpcSendResponse(&rpcMsg); + return TSDB_CODE_SUCCESS; +} int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) { SQWSchStatus *sch = NULL; @@ -801,7 +847,6 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI return TSDB_CODE_SUCCESS; } - int32_t qwHandleFetch(SQWorkerResCache *res, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; @@ -911,7 +956,6 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } - int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { @@ -1157,6 +1201,25 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + int32_t code = 0; + SVShowTablesReq *pReq = pMsg->pCont; + QW_ERR_RET(qwBuildAndSendShowRsp(pMsg, code)); +} + +int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SVShowTablesFetchReq *pFetchReq = pMsg->pCont; + QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq)); +} + int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; int8_t status = 0; @@ -1182,7 +1245,6 @@ int32_t qWorkerContinueQuery(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_RET(code); } - void qWorkerDestroy(void **qWorkerMgmt) { if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { return; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 429211bdb8..3f1799507d 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -166,7 +166,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) { } for (int32_t n = 0; n < levelPlanNum; ++n) { - SSubplan *plan = taosArrayGet(levelPlans, n); + SSubplan *plan = taosArrayGetP(levelPlans, n); SSchTask task = {0}; if (plan->type == QUERY_TYPE_MODIFY) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a923c6653..9fa5b3198b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -182,10 +182,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") -// mnode-mnode +// mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_ALREADY_EXIST, "Qnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_NOT_EXIST, "Qnode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 313f1d97af..4cb20802c7 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -16,21 +16,19 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "ulog.h" -#include "tutil.h" #include "tref.h" +#include "tutil.h" +#include "ulog.h" static int32_t tsFileRsetId = -1; static int8_t tfInited = 0; -static void tfCloseFile(void *p) { - taosCloseFile((int32_t)(uintptr_t)p); -} +static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } int32_t tfInit() { int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if(old == 1) return 0; + if (old == 1) return 0; tsFileRsetId = taosOpenRef(2000, tfCloseFile); if (tsFileRsetId > 0) { return 0; @@ -79,9 +77,7 @@ int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode return tfOpenImp(fd); } -int64_t tfClose(int64_t tfd) { - return taosRemoveRef(tsFileRsetId, tfd); -} +int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); @@ -109,6 +105,19 @@ int64_t tfRead(int64_t tfd, void *buf, int64_t count) { return ret; } +int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + + int64_t ret = pread(fd, buf, count, offset); + if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); + + taosReleaseRef(tsFileRsetId, tfd); + return ret; +} + int32_t tfFsync(int64_t tfd) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 7e7f673943..95f2fe76e6 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -85,7 +85,6 @@ int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t odbcDebugFlag = 131; int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 04bc0c8dc8..75f5e9cdbc 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -14,26 +14,29 @@ */ #include "os.h" -#include "ulog.h" + #include "taoserror.h" #include "tqueue.h" +#include "ulog.h" + +typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { - struct STaosQnode *next; - char item[]; + STaosQnode *next; + char item[]; } STaosQnode; typedef struct STaosQueue { - int32_t itemSize; - int32_t numOfItems; - struct STaosQnode *head; - struct STaosQnode *tail; - struct STaosQueue *next; // for queue set - struct STaosQset *qset; // for queue set - void *ahandle; // for queue set - FProcessItem itemFp; - FProcessItems itemsFp; - pthread_mutex_t mutex; + int32_t itemSize; + int32_t numOfItems; + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set + FProcessItem itemFp; + FProcessItems itemsFp; + pthread_mutex_t mutex; } STaosQueue; typedef struct STaosQset { @@ -52,8 +55,8 @@ typedef struct STaosQall { int32_t numOfItems; } STaosQall; -taos_queue taosOpenQueue() { - STaosQueue *queue = (STaosQueue *)calloc(sizeof(STaosQueue), 1); +STaosQueue *taosOpenQueue() { + STaosQueue *queue = calloc(sizeof(STaosQueue), 1); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -65,16 +68,14 @@ taos_queue taosOpenQueue() { return queue; } -void taosSetQueueFp(taos_queue param, FProcessItem itemFp, FProcessItems itemsFp) { - if (param == NULL) return; - STaosQueue *queue = (STaosQueue *)param; +void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp) { + if (queue == NULL) return; queue->itemFp = itemFp; queue->itemsFp = itemsFp; } -void taosCloseQueue(taos_queue param) { - if (param == NULL) return; - STaosQueue *queue = (STaosQueue *)param; +void taosCloseQueue(STaosQueue *queue) { + if (queue == NULL) return; STaosQnode *pTemp; STaosQset *qset; @@ -98,9 +99,8 @@ void taosCloseQueue(taos_queue param) { uTrace("queue:%p is closed", queue); } -bool taosQueueEmpty(taos_queue param) { - if (param == NULL) return true; - STaosQueue *queue = (STaosQueue *)param; +bool taosQueueEmpty(STaosQueue *queue) { + if (queue == NULL) return true; bool empty = false; pthread_mutex_lock(&queue->mutex); @@ -112,7 +112,7 @@ bool taosQueueEmpty(taos_queue param) { return empty; } -void *taosAllocateQitem(int size) { +void *taosAllocateQitem(int32_t size) { STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1); if (pNode == NULL) return NULL; @@ -129,9 +129,8 @@ void taosFreeQitem(void *param) { free(temp); } -int taosWriteQitem(taos_queue param, void *item) { - STaosQueue *queue = (STaosQueue *)param; - STaosQnode *pNode = (STaosQnode *)(((char *)item) - sizeof(STaosQnode)); +int32_t taosWriteQitem(STaosQueue *queue, void *pItem) { + STaosQnode *pNode = (STaosQnode *)(((char *)pItem) - sizeof(STaosQnode)); pNode->next = NULL; pthread_mutex_lock(&queue->mutex); @@ -146,7 +145,7 @@ int taosWriteQitem(taos_queue param, void *item) { queue->numOfItems++; if (queue->qset) atomic_add_fetch_32(&queue->qset->numOfItems, 1); - uTrace("item:%p is put into queue:%p, items:%d", item, queue, queue->numOfItems); + uTrace("item:%p is put into queue:%p, items:%d", pItem, queue, queue->numOfItems); pthread_mutex_unlock(&queue->mutex); @@ -155,22 +154,21 @@ int taosWriteQitem(taos_queue param, void *item) { return 0; } -int taosReadQitem(taos_queue param, void **pitem) { - STaosQueue *queue = (STaosQueue *)param; +int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { STaosQnode *pNode = NULL; - int code = 0; + int32_t code = 0; pthread_mutex_lock(&queue->mutex); if (queue->head) { pNode = queue->head; - *pitem = pNode->item; + *ppItem = pNode->item; queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); code = 1; - uDebug("item:%p is read out from queue:%p, items:%d", *pitem, queue, queue->numOfItems); + uDebug("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); @@ -178,18 +176,13 @@ int taosReadQitem(taos_queue param, void **pitem) { return code; } -void *taosAllocateQall() { - void *p = calloc(sizeof(STaosQall), 1); - return p; -} +STaosQall *taosAllocateQall() { return calloc(sizeof(STaosQall), 1); } -void taosFreeQall(void *param) { free(param); } +void taosFreeQall(STaosQall *qall) { free(qall); } -int taosReadAllQitems(taos_queue param, taos_qall p2) { - STaosQueue *queue = (STaosQueue *)param; - STaosQall *qall = (STaosQall *)p2; - int code = 0; - bool empty; +int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall) { + int32_t code = 0; + bool empty; pthread_mutex_lock(&queue->mutex); @@ -219,29 +212,25 @@ int taosReadAllQitems(taos_queue param, taos_qall p2) { return code; } -int taosGetQitem(taos_qall param, void **pitem) { - STaosQall *qall = (STaosQall *)param; +int32_t taosGetQitem(STaosQall *qall, void **ppItem) { STaosQnode *pNode; - int num = 0; + int32_t num = 0; pNode = qall->current; if (pNode) qall->current = pNode->next; if (pNode) { - *pitem = pNode->item; + *ppItem = pNode->item; num = 1; - uTrace("item:%p is fetched", *pitem); + uTrace("item:%p is fetched", *ppItem); } return num; } -void taosResetQitems(taos_qall param) { - STaosQall *qall = (STaosQall *)param; - qall->current = qall->start; -} +void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } -taos_qset taosOpenQset() { +STaosQset *taosOpenQset() { STaosQset *qset = (STaosQset *)calloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -255,9 +244,8 @@ taos_qset taosOpenQset() { return qset; } -void taosCloseQset(taos_qset param) { - if (param == NULL) return; - STaosQset *qset = (STaosQset *)param; +void taosCloseQset(STaosQset *qset) { + if (qset == NULL) return; // remove all the queues from qset pthread_mutex_lock(&qset->mutex); @@ -279,16 +267,12 @@ void taosCloseQset(taos_qset param) { // tsem_post 'qset->sem', so that reader threads waiting for it // resumes execution and return, should only be used to signal the // thread to exit. -void taosQsetThreadResume(taos_qset param) { - STaosQset *qset = (STaosQset *)param; +void taosQsetThreadResume(STaosQset *qset) { uDebug("qset:%p, it will exit", qset); tsem_post(&qset->sem); } -int taosAddIntoQset(taos_qset p1, taos_queue p2, void *ahandle) { - STaosQueue *queue = (STaosQueue *)p2; - STaosQset *qset = (STaosQset *)p1; - +int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle) { if (queue->qset) return -1; pthread_mutex_lock(&qset->mutex); @@ -309,10 +293,7 @@ int taosAddIntoQset(taos_qset p1, taos_queue p2, void *ahandle) { return 0; } -void taosRemoveFromQset(taos_qset p1, taos_queue p2) { - STaosQueue *queue = (STaosQueue *)p2; - STaosQset *qset = (STaosQset *)p1; - +void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *tqueue = NULL; pthread_mutex_lock(&qset->mutex); @@ -353,18 +334,17 @@ void taosRemoveFromQset(taos_qset p1, taos_queue p2) { uTrace("queue:%p is removed from qset:%p", queue, qset); } -int taosGetQueueNumber(taos_qset param) { return ((STaosQset *)param)->numOfQueues; } +int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } -int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProcessItem *itemFp) { - STaosQset *qset = (STaosQset *)param; +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp) { STaosQnode *pNode = NULL; - int code = 0; + int32_t code = 0; tsem_wait(&qset->sem); pthread_mutex_lock(&qset->mutex); - for (int i = 0; i < qset->numOfQueues; ++i) { + for (int32_t i = 0; i < qset->numOfQueues; ++i) { if (qset->current == NULL) qset->current = qset->head; STaosQueue *queue = qset->current; if (queue) qset->current = queue->next; @@ -375,7 +355,7 @@ int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProces if (queue->head) { pNode = queue->head; - *pitem = pNode->item; + *ppItem = pNode->item; if (ahandle) *ahandle = queue->ahandle; if (itemFp) *itemFp = queue->itemFp; queue->head = pNode->next; @@ -383,7 +363,7 @@ int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProces queue->numOfItems--; atomic_sub_fetch_32(&qset->numOfItems, 1); code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *pitem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); @@ -395,18 +375,15 @@ int taosReadQitemFromQset(taos_qset param, void **pitem, void **ahandle, FProces return code; } -int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **ahandle, FProcessItems *itemsFp) { - STaosQset *qset = (STaosQset *)param; +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp) { STaosQueue *queue; - STaosQall *qall = (STaosQall *)p2; - int code = 0; + int32_t code = 0; tsem_wait(&qset->sem); pthread_mutex_lock(&qset->mutex); - for(int i=0; inumOfQueues; ++i) { - if (qset->current == NULL) - qset->current = qset->head; + for (int32_t i = 0; i < qset->numOfQueues; ++i) { + if (qset->current == NULL) qset->current = qset->head; queue = qset->current; if (queue) qset->current = queue->next; if (queue == NULL) break; @@ -427,34 +404,32 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **ahandle, FPr queue->tail = NULL; queue->numOfItems = 0; atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); - for (int j=1; jnumOfItems; ++j) tsem_wait(&qset->sem); - } + for (int32_t j = 1; j < qall->numOfItems; ++j) tsem_wait(&qset->sem); + } pthread_mutex_unlock(&queue->mutex); - if (code != 0) break; + if (code != 0) break; } pthread_mutex_unlock(&qset->mutex); return code; } -int taosGetQueueItemsNumber(taos_queue param) { - STaosQueue *queue = (STaosQueue *)param; +int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; - int num; + int32_t num; pthread_mutex_lock(&queue->mutex); num = queue->numOfItems; pthread_mutex_unlock(&queue->mutex); return num; } -int taosGetQsetItemsNumber(taos_qset param) { - STaosQset *qset = (STaosQset *)param; +int32_t taosGetQsetItemsNumber(STaosQset *qset) { if (!qset) return 0; - int num = 0; + int32_t num = 0; pthread_mutex_lock(&qset->mutex); num = qset->numOfItems; pthread_mutex_unlock(&qset->mutex); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index fb7b71b845..ed74041712 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -85,9 +85,9 @@ static void *tWorkerThreadFp(SWorker *worker) { return NULL; } -taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) { +STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) { pthread_mutex_lock(&pool->mutex); - taos_queue queue = taosOpenQueue(); + STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); return NULL; @@ -121,7 +121,7 @@ taos_queue tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) return queue; } -void tWorkerFreeQueue(SWorkerPool *pool, void *queue) { +void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } @@ -195,11 +195,11 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) { return NULL; } -taos_queue tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) { +STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) { pthread_mutex_lock(&pool->mutex); SMWorker *worker = pool->workers + pool->nextId; - taos_queue *queue = taosOpenQueue(); + STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); return NULL; @@ -250,7 +250,7 @@ taos_queue tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems f return queue; } -void tMWorkerFreeQueue(SMWorkerPool *pool, taos_queue queue) { +void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index d31fcfb7ef..ac1bae2434 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -154,9 +154,9 @@ void acquireRleaseTest() { int32_t code = 0; int32_t num = 0; TESTSTRUCT data = {0}; - char *str1 = "abcdefg"; - char *str2 = "aaaaaaa"; - char *str3 = "123456789"; + const char *str1 = "abcdefg"; + const char *str2 = "aaaaaaa"; + const char *str3 = "123456789"; data.p = (char *)malloc(10); strcpy(data.p, str1); diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 0ccc85c347..fcc11ca213 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -120,6 +120,7 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG +echo "supportVnodes 16" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG diff --git a/tests/script/tmp/dnodes.sim b/tests/script/tmp/dnodes.sim index f13f6026f9..a3f9a0c173 100644 --- a/tests/script/tmp/dnodes.sim +++ b/tests/script/tmp/dnodes.sim @@ -1,8 +1,5 @@ -system sh/stop_dnodes.sh - - ############## config parameter ##################### -$node1 = 192.168.0.201 +$node1 = 192.168.0.201 $node2 = 192.168.0.202 $node3 = 192.168.0.203 $node4 = 192.168.0.204 @@ -10,54 +7,75 @@ $node4 = 192.168.0.204 $self = $node1 $num = 25 -############### deploy firstEp ##################### +#deploy = 0, start = 1, stop = 2 +$option = 0 +print =============== option:$option + + +############### stop dnodes ##################### +if $option == 0 then + system sh/stop_dnodes.sh +endi + +############### process firstEp ##################### $firstEp = $node1 . :7100 $firstPort = 7100 if $self == $node1 then - system sh/deploy.sh -n dnode1 -i 1 - system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp - system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp - system sh/cfg.sh -n dnode1 -c fqdn -v $node1 - system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort + if $option == 1 then + system sh/exec.sh -n dnode1 -s start + endi - system sh/exec.sh -n dnode1 -s start - sql connect + if $option == 2 then + system sh/exec.sh -n dnode1 -s stop -x SIGINT + endi + + if $option == 0 then + system sh/deploy.sh -n dnode1 -i 1 + system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp + system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp + system sh/cfg.sh -n dnode1 -c fqdn -v $node1 + system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort + system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node1 port $port - endw + system sh/exec.sh -n dnode1 -s start + sql connect - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node2 port $port - endw + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node1 port $port + endw - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node3 port $port - endw + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node2 port $port + endw - $i = 0 - while $i < $num - $port = $i * 100 - $port = $port + 8100 - $i = $i + 1 - sql create dnode $node4 port $port - endw + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node3 port $port + endw + + $i = 0 + while $i < $num + $port = $i * 100 + $port = $port + 8100 + $i = $i + 1 + sql create dnode $node4 port $port + endw + endi endi -############### deploy nodes ##################### +############### process nodes ##################### $i = 0 while $i < $num @@ -67,11 +85,21 @@ while $i < $num $dnodename = dnode . $index $i = $i + 1 - system sh/deploy.sh -n $dnodename -i 1 - system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp - system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp - system sh/cfg.sh -n $dnodename -c fqdn -v $self - system sh/cfg.sh -n $dnodename -c serverPort -v $port + if $option == 1 then + system sh/exec.sh -n $dnodename -s start + endi - system sh/exec.sh -n $dnodename -s start + if $option == 2 then + system sh/exec.sh -n $dnodename -s stop -x SIGINT + endi + + if $option == 0 then + system sh/deploy.sh -n $dnodename -i 1 + system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp + system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp + system sh/cfg.sh -n $dnodename -c fqdn -v $self + system sh/cfg.sh -n $dnodename -c serverPort -v $port + + system sh/exec.sh -n $dnodename -s start + endi endw