diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..fc9a51af65 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile + +# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT="bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9b752d091d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { "VARIANT": "ubuntu-21.04" } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "austin.code-gnu-global", + "visualstudioexptteam.vscodeintel" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a8cb1d710..cda71fb3bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,8 +25,10 @@ if(${BUILD_TEST}) enable_testing() endif(${BUILD_TEST}) add_subdirectory(source) +add_subdirectory(tools) +add_subdirectory(tests) # docs add_subdirectory(docs) -# tests (TODO) \ No newline at end of file +# tests (TODO) diff --git a/Jenkinsfile b/Jenkinsfile index c19a720d49..5faa02d0c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,6 @@ def pre_test(){ killall -9 gdb || echo "no gdb running" killall -9 python3.8 || echo "no python program running" cd ${WKC} - git reset --hard HEAD~10 >/dev/null ''' script { if (env.CHANGE_TARGET == 'master') { @@ -75,45 +74,13 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WK} - git checkout 2.0 - ''' - } - else if(env.CHANGE_TARGET == '3.0'){ - sh ''' - cd ${WK} - git checkout 3.0 - ''' - } - else{ - sh ''' - cd ${WK} - git checkout develop - ''' - } - } - sh ''' - cd ${WK} - git pull >/dev/null - git clean -dfx export TZ=Asia/Harbin date + rm -rf debug mkdir debug cd debug cmake .. > /dev/null - make > /dev/null - make install > /dev/null + make -j4> /dev/null ''' return 1 @@ -124,11 +91,11 @@ pipeline { options { skipDefaultCheckout() } environment{ WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDinternal/community' + WKC= '/var/lib/jenkins/workspace/TDengine' } stages { stage('pre_build'){ - agent{label 'master'} + agent{label 'slave3_0'} options { skipDefaultCheckout() } when { changeRequest() diff --git a/cmake/cmake.options b/cmake/cmake.options index edaab3bd45..44fa8c7e4b 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -37,12 +37,25 @@ option( off ) + option( BUILD_WITH_NURAFT "If build with NuRaft" OFF ) +option( + BUILD_WITH_UV + "If build with libuv" + OFF +) + +option( + BUILD_WITH_CRAFT + "If build with canonical-raft" + OFF +) + option( BUILD_DEPENDENCY_TESTS "If build dependency tests" @@ -54,3 +67,9 @@ option( "If use doxygen build documents" OFF ) + +option( + BUILD_WITH_INVERTEDINDEX + "If use invertedIndex" + ON +) diff --git a/cmake/craft_CMakeLists.txt.in b/cmake/craft_CMakeLists.txt.in new file mode 100644 index 0000000000..3a951c1c99 --- /dev/null +++ b/cmake/craft_CMakeLists.txt.in @@ -0,0 +1,14 @@ + +# canonical-raft +ExternalProject_Add(craft + GIT_REPOSITORY https://github.com/canonical/raft.git + GIT_TAG v0.11.2 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/craft" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft" + #BUILD_IN_SOURCE TRUE + # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example + BUILD_COMMAND "$(MAKE)" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in new file mode 100644 index 0000000000..ed406e089e --- /dev/null +++ b/cmake/libuv_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# libuv +ExternalProject_Add(libuv + GIT_REPOSITORY https://github.com/libuv/libuv.git + GIT_TAG v1.42.0 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index cffe164488..dfdde96172 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -34,10 +34,21 @@ if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) endif(${BUILD_WITH_ROCKSDB}) +# canonical-raft +if(${BUILD_WITH_CRAFT}) + cat("${CMAKE_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE) +endif(${BUILD_WITH_CRAFT}) + +#libuv +if(${BUILD_WITH_UV}) + cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_UV}) + # bdb if(${BUILD_WITH_BDB}) cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif(${BUILD_WITH_DBD}) +endif(${BUILD_WITH_BDB}) # sqlite if(${BUILD_WITH_SQLITE}) @@ -144,6 +155,23 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) +# CRAFT +if(${BUILD_WITH_CRAFT}) + add_library(craft STATIC IMPORTED GLOBAL) + set_target_properties(craft PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/craft/.libs/libraft.a" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft/include" + ) + # target_link_libraries(craft + # INTERFACE pthread + # ) +endif(${BUILD_WITH_CRAFT}) + +# LIBUV +if(${BUILD_WITH_UV}) + add_subdirectory(libuv) +endif(${BUILD_WITH_UV}) + # BDB if(${BUILD_WITH_BDB}) add_library(bdb STATIC IMPORTED GLOBAL) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 0a333f604c..330fe8f70f 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -15,4 +15,8 @@ if(${BUILD_WITH_SQLITE}) add_subdirectory(sqlite) endif(${BUILD_WITH_SQLITE}) +if(${BUILD_WITH_CRAFT}) + add_subdirectory(craft) +endif(${BUILD_WITH_CRAFT}) + add_subdirectory(tdev) diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt new file mode 100644 index 0000000000..e0f6ae64bd --- /dev/null +++ b/contrib/test/craft/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(simulate_vnode "simulate_vnode.c") +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/craft/clear.sh b/contrib/test/craft/clear.sh new file mode 100644 index 0000000000..6412656d77 --- /dev/null +++ b/contrib/test/craft/clear.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf 127.0.0.1* diff --git a/contrib/test/craft/help.txt b/contrib/test/craft/help.txt new file mode 100644 index 0000000000..48ce9de403 --- /dev/null +++ b/contrib/test/craft/help.txt @@ -0,0 +1,15 @@ + + +make craft: + +sudo apt-get install libuv1-dev liblz4-dev +autoreconf -i +./configure --enable-example +make + + +start: + +./simulate_vnode 10000 10001 10002 +./simulate_vnode 10001 10000 10002 +./simulate_vnode 10002 10000 10001 diff --git a/contrib/test/craft/simulate_vnode.c b/contrib/test/craft/simulate_vnode.c new file mode 100644 index 0000000000..668fe638b7 --- /dev/null +++ b/contrib/test/craft/simulate_vnode.c @@ -0,0 +1,252 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const char* exe_name; + +// simulate ------------------------ +typedef struct SVnode { + int vid; +} SVnode; + + +#define VNODE_COUNT 10 +SVnode vnodes[VNODE_COUNT]; + +int vnodeApplyWMsg(SVnode *pVnode, char *pMsg, void **pRsp) { + printf("put value to tsdb, vid:%d msg:%s \n", pVnode->vid, pMsg); + return 0; +} + +int applyCB(struct raft_fsm *fsm, + const struct raft_buffer *buf, + void **result) { + char *msg = (char*)buf->base; + //printf("%s \n", msg); + + // parse msg + char* context; + char* token = strtok_r(msg, ":", &context); + int vid = atoi(token); + + token = strtok_r(NULL, ":", &context); + char *value = token; + + SVnode* tmp_vnodes = (SVnode*)(fsm->data); + vnodeApplyWMsg(&tmp_vnodes[vid], value, NULL); + + return 0; +} + +// Config ------------------------ +#define HOST_LEN 32 +#define MAX_PEERS 10 +typedef struct Address { + char host[HOST_LEN]; + uint32_t port; +} Address; + +uint64_t raftId(Address *addr) { + // test in a single machine, port is unique + // if in multi machines, use host and port + return addr->port; +} + +typedef struct Config { + Address me; + Address peers[MAX_PEERS]; + int peer_count; +} Config; + +Config gConf; + +void printConf(Config *c) { + printf("me: %s:%u \n", c->me.host, c->me.port); + for (int i = 0; i < c->peer_count; ++i) { + printf("peer%d: %s:%u \n", i, c->peers[i].host, c->peers[i].port); + } +} + +// RaftServer ------------------------ +typedef struct RaftServer { + struct uv_loop_s loop; + struct raft_uv_transport transport; + struct raft_io io; + struct raft_fsm fsm; + struct raft raft; + struct raft_configuration conf; +} RaftServer; + +RaftServer gRaftServer; + +static void* startRaftServer(void *param) { + //RaftServer* rs = (RaftServer*)param; + RaftServer* rs = &gRaftServer; + raft_start(&rs->raft); + uv_run(&rs->loop, UV_RUN_DEFAULT); +} + +static const char* state2String(unsigned short state) { + if (state == RAFT_UNAVAILABLE) { + return "RAFT_UNAVAILABLE"; + + } else if (state == RAFT_FOLLOWER) { + return "RAFT_FOLLOWER"; + + } else if (state == RAFT_CANDIDATE) { + return "RAFT_CANDIDATE"; + + } else if (state == RAFT_LEADER) { + return "RAFT_LEADER"; + + } + return "UNKNOWN_RAFT_STATE"; +} + +static void printRaftState(struct raft *r) { + printf("\n"); + printf("my_id: %llu \n", r->id); + printf("address: %s \n", r->address); + printf("current_term: %llu \n", r->current_term); + printf("voted_for: %llu \n", r->voted_for); + printf("role: %s \n", state2String(r->state)); + printf("commit_index: %llu \n", r->commit_index); + printf("last_applied: %llu \n", r->last_applied); + printf("last_stored: %llu \n", r->last_stored); + printf("\n"); +} + +// console ----------------------------------------- +#define PROPOSE_VALUE_LEN 128 +static void proposeValue(struct raft *r) { + struct raft_buffer buf; + + // need free + buf.len = PROPOSE_VALUE_LEN; + buf.base = raft_malloc(buf.len); + + // mock ts value + int vid = rand() % VNODE_COUNT; + snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL)); + + printf("propose value: %s \n", (char*)buf.base); + + // need free + struct raft_apply *req = raft_malloc(sizeof(struct raft_apply)); + raft_apply(r, req, &buf, 1, NULL); +} + +static void* console(void *param) { + while (1) { + // notice! memory buffer overflow! + char buf[128]; + memset(buf, 0, sizeof(buf)); + fgets(buf, 128, stdin); + if (strlen(buf) == 1) { + continue; + } + buf[strlen(buf)-1] = '\0'; + + // do not use strcmp + if (strcmp(buf, "state") == 0) { + printRaftState(&gRaftServer.raft); + + } else if (strcmp(buf, "put") == 0) { + proposeValue(&gRaftServer.raft); + + } else { + printf("unknown command: [%s], support command: state, put \n", buf); + } + } +} + +// ----------------------------------------- +void usage() { + printf("\n"); + printf("%s my_port peer1_port peer2_port ... \n", exe_name); + printf("\n"); +} + +int main(int argc, char **argv) { + srand(time(NULL)); + + exe_name = argv[0]; + if (argc < 2) { + usage(); + exit(-1); + } + + // read conf from argv + strncpy(gConf.me.host, "127.0.0.1", HOST_LEN); + sscanf(argv[1], "%u", &gConf.me.port); + + gConf.peer_count = 0; + for (int i = 2; i < argc; ++i) { + strncpy(gConf.peers[gConf.peer_count].host, "127.0.0.1", HOST_LEN); + sscanf(argv[i], "%u", &gConf.peers[gConf.peer_count].port); + gConf.peer_count++; + } + printConf(&gConf); + + // mkdir + char dir[128]; + snprintf(dir, sizeof(dir), "./%s_%u", gConf.me.host, gConf.me.port); + + char cmd[128]; + snprintf(cmd, sizeof(cmd), "rm -rf ./%s", dir); + system(cmd); + snprintf(cmd, sizeof(cmd), "mkdir -p ./%s", dir); + system(cmd); + + // init io + uv_loop_init(&gRaftServer.loop); + raft_uv_tcp_init(&gRaftServer.transport, &gRaftServer.loop); + raft_uv_init(&gRaftServer.io, &gRaftServer.loop, dir, &gRaftServer.transport); + + // init fsm + gRaftServer.fsm.apply = applyCB; + gRaftServer.fsm.data = vnodes; + for (int i = 0; i < VNODE_COUNT; ++i) { + vnodes[i].vid = i; + } + + // init raft instance with io and fsm + char address_buf[128]; + snprintf(address_buf, sizeof(address_buf), "%s:%u", gConf.me.host, gConf.me.port); + + // test in a single machine, port is unique + uint64_t raft_id = raftId(&gConf.me); + raft_init(&gRaftServer.raft, &gRaftServer.io, &gRaftServer.fsm, raft_id, address_buf); + //raft_init(&gRaftServer.raft, &gRaftServer.io, &gRaftServer.fsm, 11, "127.0.0.1:9000"); + + // init cluster configuration + struct raft_configuration conf; + raft_configuration_init(&conf); + raft_configuration_add(&conf, raftId(&gConf.me), address_buf, RAFT_VOTER); + for (int i = 0; i < gConf.peer_count; ++i) { + char address_buf[128]; + snprintf(address_buf, sizeof(address_buf), "%s:%u", gConf.peers[i].host, gConf.peers[i].port); + raft_configuration_add(&conf, raftId(&gConf.peers[i]), address_buf, RAFT_VOTER); + } + raft_bootstrap(&gRaftServer.raft, &conf); + + // start raft server and loop + pthread_t tid; + pthread_create(&tid, NULL, startRaftServer, &gRaftServer); + + // simulate console + pthread_t tid2; + pthread_create(&tid2, NULL, console, NULL); + + while (1) { + sleep(10); + } + + return 0; +} diff --git a/include/client/taos.h b/include/client/taos.h index 19d191b84e..7357478555 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -39,7 +39,7 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes #define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes #define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_BINARY 8 // string, alias for varchar #define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes #define TSDB_DATA_TYPE_NCHAR 10 // unicode string #define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte @@ -47,10 +47,10 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_UINT 13 // 4 bytes #define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes #define TSDB_DATA_TYPE_VARCHAR 15 // string -#define TSDB_DATA_TYPE_JSON 16 // json -#define TSDB_DATA_TYPE_DECIMAL 17 // decimal -#define TSDB_DATA_TYPE_BLOB 18 // binary string -#define TSDB_DATA_TYPE_LONGBLOB 19 // long binary string +#define TSDB_DATA_TYPE_VARBINARY 16 // binary +#define TSDB_DATA_TYPE_JSON 17 // json +#define TSDB_DATA_TYPE_DECIMAL 18 // decimal +#define TSDB_DATA_TYPE_BLOB 19 // binary typedef enum { TSDB_OPTION_LOCALE, @@ -61,10 +61,27 @@ typedef enum { TSDB_MAX_OPTIONS } TSDB_OPTION; +typedef enum { + TSDB_SML_UNKNOWN_PROTOCOL = 0, + TSDB_SML_LINE_PROTOCOL = 1, + TSDB_SML_TELNET_PROTOCOL = 2, + TSDB_SML_JSON_PROTOCOL = 3, +} TSDB_SML_PROTOCOL_TYPE; + +typedef enum { + TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + TSDB_SML_TIMESTAMP_HOURS, + TSDB_SML_TIMESTAMP_MINUTES, + TSDB_SML_TIMESTAMP_SECONDS, + TSDB_SML_TIMESTAMP_MILLI_SECONDS, + TSDB_SML_TIMESTAMP_MICRO_SECONDS, + TSDB_SML_TIMESTAMP_NANO_SECONDS, +} TSDB_SML_TIMESTAMP_TYPE; + typedef struct taosField { char name[65]; - uint8_t type; - int16_t bytes; + int8_t type; + int32_t bytes; } TAOS_FIELD; #ifdef _TD_GO_DLL_ @@ -136,37 +153,36 @@ DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); + DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); -DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); -DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); +DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); -// TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); -// TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); - -// TODO: the return value should be `const` DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); -DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); -DLL_EXPORT int taos_errno(TAOS_RES *tres); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); +typedef void (*__taos_sub_fn_t)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); +DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, __taos_sub_fn_t fp, void *param, int interval); DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); @@ -175,8 +191,7 @@ DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp) DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); - -DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines); +DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); #ifdef __cplusplus } diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 55f51c285c..a788df2e32 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -51,6 +51,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -214,8 +215,8 @@ typedef enum _mgmt_table { extern char *taosMsg[]; typedef struct SBuildTableMetaInput { - int32_t vgId; - char *tableFullName; + int32_t vgId; + char *tableFullName; } SBuildTableMetaInput; typedef struct SBuildUseDBInput { @@ -233,9 +234,9 @@ typedef struct { } SEpAddrMsg; typedef struct { - char* fqdn; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SEpAddr1; +} SEpAddr; typedef struct { int32_t numOfVnodes; @@ -260,10 +261,10 @@ typedef struct SSubmitBlk { // Submit message for this TSDB typedef struct SSubmitMsg { - SMsgHead header; - int32_t length; - int32_t numOfBlocks; - char blocks[]; + SMsgHead header; + int32_t length; + int32_t numOfBlocks; + char blocks[]; } SSubmitMsg; typedef struct { @@ -351,17 +352,17 @@ typedef struct { } SAlterTableMsg; typedef struct { - SMsgHead head; - int64_t uid; - int32_t tid; - int16_t tversion; - int16_t colId; - int8_t type; - int16_t bytes; - int32_t tagValLen; - int16_t numOfTags; - int32_t schemaLen; - char data[]; + SMsgHead head; + int64_t uid; + int32_t tid; + int16_t tversion; + int16_t colId; + int8_t type; + int16_t bytes; + int32_t tagValLen; + int16_t numOfTags; + int32_t schemaLen; + char data[]; } SUpdateTableTagValMsg; typedef struct { @@ -405,8 +406,10 @@ typedef struct { } SDropUserMsg, SDropAcctMsg; typedef struct { + int8_t type; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; + int8_t superUser; // denote if it is a super user or not int32_t reserve[8]; } SCreateUserMsg, SAlterUserMsg; @@ -422,7 +425,7 @@ typedef struct { int32_t contLen; int32_t vgId; uint64_t uid; - char tableFname[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SDropSTableMsg; typedef struct SColIndex { @@ -435,7 +438,7 @@ typedef struct SColIndex { typedef struct SColumnFilterInfo { int16_t lowerRelOptr; int16_t upperRelOptr; - int16_t filterstr; // denote if current column is char(binary/nchar) + int16_t filterstr; // denote if current column is char(binary/nchar) union { struct { @@ -454,9 +457,9 @@ typedef struct SColumnFilterInfo { } SColumnFilterInfo; typedef struct SColumnFilterList { - int16_t numOfFilters; - union{ - int64_t placeholder; + int16_t numOfFilters; + union { + int64_t placeholder; SColumnFilterInfo *filterInfo; }; } SColumnFilterList; @@ -465,10 +468,10 @@ typedef struct SColumnFilterList { * But for data in vnode side, we need all the following information. */ typedef struct SColumnInfo { - int16_t colId; - int16_t type; - int16_t bytes; - SColumnFilterList flist; + int16_t colId; + int16_t type; + int16_t bytes; + SColumnFilterList flist; } SColumnInfo; typedef struct STableIdInfo { @@ -482,14 +485,14 @@ typedef struct STimeWindow { } STimeWindow; typedef struct { - int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed - int32_t tsLen; // total length of ts comp block - int32_t tsNumOfBlocks; // ts comp block numbers - int32_t tsOrder; // ts comp block order + int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed + int32_t tsLen; // total length of ts comp block + int32_t tsNumOfBlocks; // ts comp block numbers + int32_t tsOrder; // ts comp block order } STsBufInfo; typedef struct SInterval { - int32_t tz; // query client timezone + int32_t tz; // query client timezone char intervalUnit; char slidingUnit; char offsetUnit; @@ -499,48 +502,48 @@ typedef struct SInterval { } SInterval; typedef struct { - SMsgHead head; - char version[TSDB_VERSION_LEN]; + SMsgHead head; + char version[TSDB_VERSION_LEN]; - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool interpQuery; // interp query or not - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryBlockDist; // if query data block distribution - bool stabledev; // super table stddev query - bool tsCompQuery; // is tscomp query - bool simpleAgg; - bool pointInterpQuery; // point interpolation query - bool needReverseScan; // need reverse scan - bool stateWindow; // state window flag + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool interpQuery; // interp query or not + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo; // if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + bool tsCompQuery; // is tscomp query + bool simpleAgg; + bool pointInterpQuery; // point interpolation query + bool needReverseScan; // need reverse scan + bool stateWindow; // state window flag STimeWindow window; int32_t numOfTables; int16_t order; int16_t orderColId; - int16_t numOfCols; // the number of columns will be load from vnode + int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; -// SSessionWindow sw; // session window - int16_t tagCondLen; // tag length in current query - int16_t colCondLen; // column length in current query - int16_t numOfGroupCols; // num of group by columns - int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx - int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. - int16_t prjOrder; // global order in super table projection query. - int64_t limit; - int64_t offset; - int32_t queryType; // denote another query process - int16_t numOfOutput; // final output columns numbers - int16_t fillType; // interpolate type - int64_t fillVal; // default value array list - int32_t secondStageOutput; - STsBufInfo tsBuf; // tsBuf info - int32_t numOfTags; // number of tags columns involved - int32_t sqlstrLen; // sql query string - int32_t prevResultLen; // previous result length + // SSessionWindow sw; // session window + int16_t tagCondLen; // tag length in current query + int16_t colCondLen; // column length in current query + int16_t numOfGroupCols; // num of group by columns + int16_t orderByIdx; + int16_t orderType; // used in group by xx order by xxx + int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int16_t prjOrder; // global order in super table projection query. + int64_t limit; + int64_t offset; + int32_t queryType; // denote another query process + int16_t numOfOutput; // final output columns numbers + int16_t fillType; // interpolate type + int64_t fillVal; // default value array list + int32_t secondStageOutput; + STsBufInfo tsBuf; // tsBuf info + int32_t numOfTags; // number of tags columns involved + int32_t sqlstrLen; // sql query string + int32_t prevResultLen; // previous result length int32_t numOfOperator; int32_t tableScanOperator;// table scan operator. -1 means no scan operator int32_t udfNum; // number of udf function @@ -569,19 +572,19 @@ typedef struct { } SRetrieveTableMsg; typedef struct SRetrieveTableRsp { - int32_t numOfRows; - int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; int8_t completed; // all results are returned to client int8_t precision; int8_t compressed; - int8_t reserved; int32_t compLen; + + int32_t numOfRows; char data[]; } SRetrieveTableRsp; typedef struct { char db[TSDB_FULL_DB_NAME_LEN]; + int32_t numOfVgroups; int32_t cacheBlockSize; // MB int32_t totalBlocks; int32_t daysPerFile; @@ -624,7 +627,6 @@ typedef struct { typedef struct { char db[TSDB_TABLE_FNAME_LEN]; - int8_t ignoreNotExists; int32_t vgVersion; int32_t reserve[8]; } SUseDbMsg; @@ -750,31 +752,36 @@ typedef struct { } SReplica; typedef struct { - char db[TSDB_FULL_DB_NAME_LEN]; int32_t vgId; + int32_t dnodeId; + char db[TSDB_FULL_DB_NAME_LEN]; + uint64_t dbUid; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; + int32_t minRows; + int32_t maxRows; + int32_t commitTime; int32_t fsyncPeriod; - int8_t reserved[16]; + int8_t walLevel; int8_t precision; int8_t compression; - int8_t cacheLastRow; - int8_t update; - int8_t walLevel; int8_t quorum; + int8_t update; + int8_t cacheLastRow; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; } SCreateVnodeMsg, SAlterVnodeMsg; typedef struct { - int32_t vgId; + int32_t vgId; + int32_t dnodeId; + char db[TSDB_FULL_DB_NAME_LEN]; + uint64_t dbUid; } SDropVnodeMsg, SSyncVnodeMsg, SCompactVnodeMsg; typedef struct { @@ -783,12 +790,8 @@ typedef struct { } SAuthVnodeMsg; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; -} SStbInfoMsg; - -typedef struct { - SMsgHead msgHead; - char tableFname[TSDB_TABLE_FNAME_LEN]; + int32_t vgId; + char tableFname[TSDB_TABLE_FNAME_LEN]; } STableInfoMsg; typedef struct { @@ -799,25 +802,15 @@ typedef struct { char tableNames[]; } SMultiTableInfoMsg; -typedef struct SSTableVgroupMsg { - int32_t numOfTables; -} SSTableVgroupMsg, SSTableVgroupRspMsg; - typedef struct SVgroupInfo { int32_t vgId; - int32_t hashBegin; - int32_t hashEnd; + uint32_t hashBegin; + uint32_t hashEnd; int8_t inUse; int8_t numOfEps; SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; -typedef struct SVgroupListRspMsg { - int32_t vgroupNum; - int32_t vgroupVersion; - SVgroupInfo vgroupInfo[]; -} SVgroupListRspMsg; - typedef struct { int32_t vgId; int8_t numOfEps; @@ -830,19 +823,19 @@ typedef struct { } SVgroupsMsg, SVgroupsInfo; typedef struct { - char tbFname[TSDB_TABLE_FNAME_LEN]; // table id - char stbFname[TSDB_TABLE_FNAME_LEN]; - int32_t numOfTags; - int32_t numOfColumns; - int8_t precision; - int8_t tableType; - int8_t update; - int32_t sversion; - int32_t tversion; - uint64_t tuid; - uint64_t suid; - int32_t vgId; - SSchema pSchema[]; + char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name + char stbFname[TSDB_TABLE_FNAME_LEN]; + int32_t numOfTags; + int32_t numOfColumns; + int8_t precision; + int8_t tableType; + int8_t update; + int32_t sversion; + int32_t tversion; + uint64_t suid; + uint64_t tuid; + int32_t vgId; + SSchema pSchema[]; } STableMetaMsg; typedef struct SMultiTableMeta { @@ -865,13 +858,11 @@ typedef struct { typedef struct { char db[TSDB_FULL_DB_NAME_LEN]; int32_t vgVersion; - int32_t vgNum; + int32_t vgNum; int8_t hashMethod; SVgroupInfo vgroupInfo[]; } SUseDbRsp; - - /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' @@ -885,7 +876,7 @@ typedef struct { } SShowMsg; typedef struct { - char db[TSDB_FULL_DB_NAME_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; int32_t numOfVgroup; int32_t vgid[]; } SCompactMsg; @@ -1006,10 +997,10 @@ typedef struct { } SAuthMsg, SAuthRsp; typedef struct { - int8_t finished; - int8_t reserved1[7]; - char name[TSDB_STEP_NAME_LEN]; - char desc[TSDB_STEP_DESC_LEN]; + int8_t finished; + int8_t reserved1[7]; + char name[TSDB_STEP_NAME_LEN]; + char desc[TSDB_STEP_DESC_LEN]; } SStartupMsg; // mq related @@ -1044,7 +1035,7 @@ typedef struct { typedef struct { } SMqResetRsp; -//mq related end +// mq related end typedef struct { /* data */ @@ -1098,6 +1089,23 @@ typedef struct { /* data */ } SUpdateTagValRsp; +typedef struct SSchedulerQueryMsg { + uint64_t queryId; + uint64_t taskId; + uint32_t contentLen; + char msg[]; +} SSchedulerQueryMsg; + +typedef struct SSchedulerReadyMsg { + uint64_t queryId; + uint64_t taskId; +} SSchedulerReadyMsg; + +typedef struct SSchedulerFetchMsg { + uint64_t queryId; + uint64_t taskId; +} SSchedulerFetchMsg; + #pragma pack(pop) diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index d357ca1f47..1fb10ae15b 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -62,7 +62,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_MNODE, "retrieve" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" ) diff --git a/include/common/tname.h b/include/common/tname.h index 7578a7804c..e31bfd38a6 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,7 +16,7 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H -#include "taosmsg.h" +//#include "taosmsg.h" #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 @@ -27,7 +27,7 @@ typedef struct SName { uint8_t type; //db_name_t, table_name_t - char acctId[TSDB_ACCT_ID_LEN]; + int32_t acctId; char dbname[TSDB_DB_NAME_LEN]; char tname[TSDB_TABLE_NAME_LEN]; } SName; @@ -38,7 +38,7 @@ int32_t tNameLen(const SName* name); SName* tNameDup(const SName* name); -bool tIsValidName(const SName* name); +bool tNameIsValid(const SName* name); const char* tNameGetTableName(const SName* name); @@ -50,14 +50,10 @@ bool tNameIsEmpty(const SName* name); void tNameAssign(SName* dst, const SName* src); +int32_t tNameSetDbName(SName* dst, int32_t acctId, const char* dbName, size_t nameLen); + int32_t tNameFromString(SName* dst, const char* str, uint32_t type); -int32_t tNameSetAcctId(SName* dst, const char* acct); - -SSchema* tGetTbnameColumnSchema(); - -#if 0 -int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); -#endif +int32_t tNameSetAcctId(SName* dst, int32_t acctId); #endif // TDENGINE_TNAME_H diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 44ae1bb79f..113a970548 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -16,6 +16,7 @@ #ifndef _TD_META_H_ #define _TD_META_H_ +#include "mallocator.h" #include "os.h" #include "trow.h" @@ -71,7 +72,7 @@ typedef struct STbCfg { } STbCfg; // SMeta operations -SMeta *metaOpen(const char *path, const SMetaCfg *pOptions); +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); void metaClose(SMeta *pMeta); void metaRemove(const char *path); int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); @@ -79,8 +80,8 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // Options -void metaOptionsInit(SMetaCfg *pOptions); -void metaOptionsClear(SMetaCfg *pOptions); +void metaOptionsInit(SMetaCfg *pMetaCfg); +void metaOptionsClear(SMetaCfg *pMetaCfg); // STbCfg #define META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \ diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index f8eac9768f..e5522ddbd3 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -16,31 +16,33 @@ #ifndef _TD_TSDB_H_ #define _TD_TSDB_H_ +#include "mallocator.h" + #ifdef __cplusplus extern "C" { #endif // TYPES EXPOSED -typedef struct STsdb STsdb; -typedef struct STsdbCfg STsdbCfg; +typedef struct STsdb STsdb; -// STsdb -STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg); -void tsdbClose(STsdb *); -void tsdbRemove(const char *path); -int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg); - -// STsdbCfg -int tsdbOptionsInit(STsdbCfg *); -void tsdbOptionsClear(STsdbCfg *); - -/* ------------------------ STRUCT DEFINITIONS ------------------------ */ -struct STsdbCfg { +typedef struct STsdbCfg { uint64_t lruCacheSize; uint32_t keep0; uint32_t keep1; uint32_t keep2; -}; +} STsdbCfg; + +// STsdb +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); +void tsdbClose(STsdb *); +void tsdbRemove(const char *path); +int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg); +int tsdbPrepareCommit(STsdb *pTsdb); +int tsdbCommit(STsdb *pTsdb); + +// STsdbCfg +int tsdbOptionsInit(STsdbCfg *); +void tsdbOptionsClear(STsdbCfg *); #ifdef __cplusplus } diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 1f2452291b..68eae03f51 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -32,8 +32,7 @@ extern "C" { struct SCatalog; typedef struct SCatalogReq { - char dbName[TSDB_DB_NAME_LEN]; - SArray *pTableName; // table full name + SArray *pTableName; // element is SNAME SArray *pUdf; // udf name bool qNodeRequired; // valid qnode } SCatalogReq; @@ -54,10 +53,10 @@ typedef struct SCatalogCfg { int32_t catalogInit(SCatalogCfg *cfg); /** - * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. - * There is ONLY one SCatalog object for one process space, and this function returns a singleton. - * @param clusterId - * @return + * Get a cluster's catalog handle for all later operations. + * @param clusterId (input, end with \0) + * @param catalogHandle (output, NO need to free it) + * @return error code */ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); @@ -65,40 +64,85 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo); int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); - +/** + * Get a table's meta data. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @return error code + */ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); + +/** + * Force renew a table's local cached meta data. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @return error code + */ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName); + +/** + * Force renew a table's local cached meta data and get the new one. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @return error code + */ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); /** - * get table's vgroup list. - * @param clusterId - * @pVgroupList - array of SVgroupInfo - * @return + * Get a table's actual vgroup, for stable it's all possible vgroup list. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) + * @return error code */ -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); +int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); + +/** + * Get a table's vgroup from its name's hash value. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param vgInfo (output, vgroup info) + * @return error code + */ +int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo); /** - * Get the required meta data from mnode. - * Note that this is a synchronized API and is also thread-safety. - * @param pCatalog - * @param pMgmtEps - * @param pMetaReq - * @param pMetaData - * @return + * Get all meta data required in pReq. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pReq (input, reqest info) + * @param pRsp (output, response data) + * @return error code */ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); /** * Destroy catalog and relase all resources - * @param pCatalog */ void catalogDestroy(void); diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 1b74928568..2535ec8a5b 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -24,10 +24,20 @@ extern "C" { #endif typedef struct SIndex SIndex; +typedef struct SIndexTerm SIndexTerm; typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; +typedef enum { + ADD_VALUE, // add index colume value + DEL_VALUE, // delete index column value + UPDATE_VALUE, // update index column value + ADD_INDEX, // add index on specify column + DROP_INDEX, // drop existed index + DROP_SATBLE // drop stable +} SIndexOperOnColumn; + typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; /* @@ -36,14 +46,14 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = */ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type); +int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); /* * @param: * @param: */ -SIndex* indexOpen(SIndexOpts *opt, const char *path); +int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); void indexClose(SIndex *index); -int indexPut(SIndex *index, SIndexMultiTerm *terms, int uid); +int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); int indexDelete(SIndex *index, SIndexMultiTermQuery *query); int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); int indexRebuild(SIndex *index, SIndexOpts *opt); @@ -52,8 +62,8 @@ int indexRebuild(SIndex *index, SIndexOpts *opt); * @param */ SIndexMultiTerm *indexMultiTermCreate(); -int indexMultiTermAdd(SIndexMultiTerm *terms, const char *field, int32_t nFields, const char *value, int32_t nValue); -void indexMultiTermDestroy(SIndexMultiTerm *terms); +int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); +void indexMultiTermDestroy(SIndexMultiTerm *terms); /* * @param: * @param: @@ -61,6 +71,16 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms); SIndexOpts *indexOptsCreate(); void indexOptsDestroy(SIndexOpts *opts); +/* + * @param: + * @param: + */ + +SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, + const char *colName, int32_t nColName, const char *colVal, int32_t nColVal); +void indexTermDestroy(SIndexTerm *p); + + #ifdef __cplusplus } #endif diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index d65b5ab570..8e26fa98c9 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -153,7 +153,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen); +int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); typedef enum { PAYLOAD_TYPE_KV = 0, @@ -209,6 +209,7 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); +int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 844757eeb5..8b54b88b28 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -34,6 +34,13 @@ enum OPERATOR_TYPE_E { OP_TotalNum }; +enum DATASINK_TYPE_E { + DSINK_Unknown, + DSINK_Dispatch, + DSINK_Insert, + DSINK_TotalNum +}; + struct SEpSet; struct SQueryStmtInfo; @@ -49,6 +56,22 @@ typedef struct SQueryNodeBasicInfo { const char *name; // operator name } SQueryNodeBasicInfo; +typedef struct SDataSink { + SQueryNodeBasicInfo info; + SDataBlockSchema schema; +} SDataSink; + +typedef struct SDataDispatcher { + SDataSink sink; + // todo +} SDataDispatcher; + +typedef struct SDataInserter { + SDataSink sink; + uint64_t uid; // unique id of the table + // todo data field +} SDataInserter; + typedef struct SPhyNode { SQueryNodeBasicInfo info; SArray *pTargets; // target list to be computed or scanned at this node @@ -84,8 +107,8 @@ typedef struct SProjectPhyNode { typedef struct SExchangePhyNode { SPhyNode node; - uint64_t templateId; - SArray *pSourceEpSet; // SEpSet + uint64_t srcTemplateId; // template id of datasource suplans + SArray *pSrcEndPoints; // SEpAddrMsg, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; typedef struct SSubplanId { @@ -105,6 +128,8 @@ typedef struct SSubplan { } SSubplan; typedef struct SQueryDag { + uint64_t queryId; + int32_t numOfSubplans; SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; @@ -113,20 +138,29 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); -int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); +// Set datasource of this subplan, multiple calls may be made to a subplan. +// @subplan subplan to be schedule +// @templateId templateId of a group of datasource subplans of this @subplan +// @ep one execution location of this group of datasource subplans +int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); +int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); /** * Convert to subplan to string for the scheduler to send to the executor */ -int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str); +int32_t qSubPlanToString(const SSubplan* subplan, char** str); + +int32_t qStringToSubplan(const char* str, SSubplan** subplan); + +void qDestroySubplan(SSubplan* pSubplan); /** * Destroy the physical plan. * @param pQueryPhyNode * @return */ -void* qDestroyQueryDag(struct SQueryDag* pDag); +void qDestroyQueryDag(SQueryDag* pDag); #ifdef __cplusplus } diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 27c7c534a2..41d6e028cf 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -14,20 +14,20 @@ */ #if defined(INCLUDE_AS_ENUM) // enum define mode - #undef OP_ENUM_MACRO - #define OP_ENUM_MACRO(op) OP_##op, + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) OP_##op, #elif defined(INCLUDE_AS_NAME) // comment define mode - #undef OP_ENUM_MACRO - #define OP_ENUM_MACRO(op) #op, + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) #op, #else - #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME + #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME #endif OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) -OP_ENUM_MACRO(TableBlockInfoScan) +OP_ENUM_MACRO(SystemTableScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) OP_ENUM_MACRO(Groupby) diff --git a/include/libs/query/query.h b/include/libs/qcom/query.h similarity index 72% rename from include/libs/query/query.h rename to include/libs/qcom/query.h index 8720fd085c..254d572149 100644 --- a/include/libs/query/query.h +++ b/include/libs/qcom/query.h @@ -22,8 +22,7 @@ extern "C" { #include "tarray.h" #include "thash.h" - -typedef SVgroupListRspMsg SVgroupListInfo; +#include "tlog.h" typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema @@ -82,9 +81,24 @@ typedef struct STableMetaOutput { STableMeta *tbMeta; } STableMetaOutput; +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); + extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); +SSchema* tGetTbnameColumnSchema(); +extern void msgInit(); + +extern int32_t qDebugFlag; + +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 6b3c9ed021..2cbf26f877 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,6 +20,13 @@ extern "C" { #endif +#include "planner.h" +#include "catalog.h" + +typedef struct SSchedulerCfg { + int32_t clusterType; +} SSchedulerCfg; + typedef struct SQueryProfileSummary { int64_t startTs; // Object created and added into the message queue int64_t endTs; // the timestamp when the task is completed @@ -43,43 +50,29 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; -typedef struct SQueryTask { - uint64_t queryId; // query id - uint64_t taskId; // task id - char *pSubplan; // operator tree - uint64_t status; // task status - SQueryProfileSummary summary; // task execution summary - void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage -} SQueryTask; - -typedef struct SQueryJob { - SArray **pSubtasks; - // todo -} SQueryJob; +int32_t schedulerInit(SSchedulerCfg *cfg); /** * Process the query job, generated according to the query physical plan. * This is a synchronized API, and is also thread-safety. - * @param pJob + * @param qnodeList Qnode address list, element is SEpAddr * @return */ -int32_t qProcessQueryJob(struct SQueryJob* pJob); +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); + +int32_t scheduleFetchRows(void *pJob, void **data); -/** - * The SSqlObj should not be here???? - * @param pSql - * @param pVgroupId - * @param pRetVgroupId - * @return - */ -//SArray* qGetInvolvedVgroupIdList(struct SSqlObj* pSql, SArray* pVgroupId, SArray* pRetVgroupId); /** * Cancel query job * @param pJob * @return */ -int32_t qKillQueryJob(struct SQueryJob* pJob); +int32_t scheduleCancelJob(void *pJob); + +void scheduleFreeJob(void *pJob); + +void schedulerDestroy(void); #ifdef __cplusplus } diff --git a/include/os/os.h b/include/os/os.h index 53a6cef96a..de2a8182db 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -46,6 +46,7 @@ extern "C" { #include #include #include +#include #include #include diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 56f6b3e0da..6952b91742 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "os.h" + #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 96 @@ -57,11 +59,11 @@ char * taosGetCmdlineByPID(int pid); void taosSetCoreDump(bool enable); typedef struct { - const char *sysname; - const char *nodename; - const char *release; - const char *version; - const char *machine; + char sysname[_UTSNAME_MACHINE_LENGTH]; + char nodename[_UTSNAME_MACHINE_LENGTH]; + char release[_UTSNAME_MACHINE_LENGTH]; + char version[_UTSNAME_MACHINE_LENGTH]; + char machine[_UTSNAME_MACHINE_LENGTH]; } SysNameInfo; SysNameInfo taosGetSysNameInfo(); diff --git a/include/util/mallocator.h b/include/util/mallocator.h index ffe242017e..5ecdc316a4 100644 --- a/include/util/mallocator.h +++ b/include/util/mallocator.h @@ -22,23 +22,31 @@ extern "C" { #endif -typedef struct SMemAllocator SMemAllocator; -typedef struct SMemAllocatorFactory SMemAllocatorFactory; +// Memory allocator +#define TD_MEM_ALCT(TYPE) \ + struct { \ + void *(*malloc_)(struct TYPE *, uint64_t size); \ + void (*free_)(struct TYPE *, void *ptr); \ + } +#define TD_MA_MALLOC_FUNC(TMA) (TMA)->malloc_ +#define TD_MA_FREE_FUNC(TMA) (TMA)->free_ -struct SMemAllocator { - void *impl; - void *(*malloc)(SMemAllocator *, uint64_t size); - void *(*calloc)(SMemAllocator *, uint64_t nmemb, uint64_t size); - void *(*realloc)(SMemAllocator *, void *ptr, uint64_t size); - void (*free)(SMemAllocator *, void *ptr); - uint64_t (*usage)(SMemAllocator *); -}; +#define TD_MA_MALLOC(TMA, SIZE) (*((TMA)->malloc_))(TMA, (SIZE)) +#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR)) -struct SMemAllocatorFactory { +typedef struct SMemAllocator { void *impl; - SMemAllocator *(*create)(SMemAllocatorFactory *); - void (*destroy)(SMemAllocatorFactory *, SMemAllocator *); -}; + TD_MEM_ALCT(SMemAllocator); +} SMemAllocator; + +#define tMalloc(pMA, SIZE) TD_MA_MALLOC(PMA, SIZE) +#define tFree(pMA, PTR) TD_MA_FREE(PMA, PTR) + +typedef struct SMemAllocatorFactory { + void *impl; + SMemAllocator *(*create)(struct SMemAllocatorFactory *); + void (*destroy)(struct SMemAllocatorFactory *, SMemAllocator *); +} SMemAllocatorFactory; #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f15e4c1c97..95824df5ac 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -120,17 +120,18 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") // mnode-common -#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0300) -#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0301) -#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302) -#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0304) -#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0305) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0306) -#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0307) -#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0308) -#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0309) -#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030A) +#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) +#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301) +#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) +#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0303) +#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0304) +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0305) +#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0306) +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0307) +#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0308) +#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0309) +#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x030A) +#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030B) // mnode-show #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) @@ -206,44 +207,28 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392) // mnode-stable -#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_IGEXIST TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COLS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_TYPE TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_ID TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_BYTES TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_NAME TAOS_DEF_ERROR_CODE(0, 0x0360) - - -#define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") -#define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") -#define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") -#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) //"Too many tags") -#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x0365) //"Too many columns") -#define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) //"Too many time series") -#define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) //"Not super table") // operation only available for super table -#define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) //"Tag name too long") -#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) //"Tag already exists") -#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) //"Tag does not exist") -#define TSDB_CODE_MND_FIELD_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) //"Field already exists") -#define TSDB_CODE_MND_FIELD_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) //"Field does not exist") -#define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist") -#define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") -#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) //"Exceed max row bytes") - -#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370) -#define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371) -#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0372) -#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0373) -#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0374) -#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0375) -#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0376) - -#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") -#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") +#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) +#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A1) +#define TSDB_CODE_MND_TOO_MANY_STBS TAOS_DEF_ERROR_CODE(0, 0x03A2) +#define TSDB_CODE_MND_INVALID_STB TAOS_DEF_ERROR_CODE(0, 0x03A3) +#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03A4) +#define TSDB_CODE_MND_STB_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03A5) +#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x03A6) +#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A7) +#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A8) +#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03A9) +#define TSDB_CODE_MND_COLUMN_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) +#define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) +#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x03AC) +// mnode-func +#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) +#define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C1) +#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x03C2) +#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x03C3) +#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x03C4) +#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) +#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) @@ -324,6 +309,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") #define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition") #define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error") +#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) //"invalid input") // grant @@ -519,6 +505,12 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error #define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error +//scheduler +#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error +#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error + + + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index 85b51b6eff..f3f3643268 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -196,6 +196,7 @@ do { \ #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 @@ -230,10 +231,11 @@ do { \ #define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_CQ_SQL_SIZE 1024 -#define TSDB_MIN_VNODES 64 +#define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 -#define TSDB_MIN_VNODES_PER_DB 2 -#define TSDB_MAX_VNODES_PER_DB 64 +#define TSDB_MIN_VNODES_PER_DB 1 +#define TSDB_MAX_VNODES_PER_DB 4096 +#define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_MGMT 1 @@ -246,7 +248,7 @@ do { \ #define TSDB_RES_COL_ID (-5000) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta - + #define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 diff --git a/include/util/tdlist.h b/include/util/tdlist.h deleted file mode 100644 index d047a57770..0000000000 --- a/include/util/tdlist.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_TDLIST_H_ -#define _TD_UTIL_TDLIST_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -// Single linked list -#define TD_SLIST_NODE(TYPE) \ - struct { \ - struct TYPE *sl_next_; \ - } - -#define TD_SLIST(TYPE) \ - struct { \ - struct TYPE *sl_head_; \ - int sl_neles_; \ - } - -#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) -#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) -#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) - -#define tSListInit(sl) \ - do { \ - (sl)->sl_head_ = NULL; \ - (sl)->sl_neles_ = 0; \ - } while (0) - -#define tSListPush(sl, sln) \ - do { \ - TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ - TD_SLIST_HEAD(sl) = (sln); \ - TD_SLIST_NELES(sl) += 1; \ - } while (0) - -#define tSListPop(sl) \ - do { \ - TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ - TD_SLIST_NELES(sl) -= 1; \ - } while (0) - -// Double linked list -#define TD_DLIST_NODE(TYPE) \ - struct { \ - struct TYPE *dl_prev_; \ - struct TYPE *dl_next_; \ - } - -#define TD_DLIST(TYPE) \ - struct { \ - struct TYPE *dl_head_; \ - struct TYPE *dl_tail_; \ - int dl_neles_; \ - } - -#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) -#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) -#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) -#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) -#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) - -#define tDListInit(dl) \ - do { \ - TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ - TD_DLIST_NELES(dl) = 0; \ - } while (0) - -#define tDListAppend(dl, dln) \ - do { \ - if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ - TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ - } else { \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ - TD_DLIST_NODE_NEXT(dln) = NULL; \ - TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ - TD_DLIST_TAIL(dl) = (dln); \ - } \ - TD_DLIST_NELES(dl) += 1; \ - } while (0) - -#define tDListPrepend(dl, dln) \ - do { \ - if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ - TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ - } else { \ - TD_DLIST_NODE_PREV(dln) = NULL; \ - TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ - TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ - TD_DLIST_HEAD(dl) = (dln); \ - } \ - TD_DLIST_NELES(dl) += 1; \ - } while (0) - -#define tDListPop(dl, dln) \ - do { \ - if (TD_DLIST_HEAD(dl) == (dln)) { \ - TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ - } \ - if (TD_DLIST_TAIL(dl) == (dln)) { \ - TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ - } \ - if (TD_DLIST_NODE_PREV(dln) != NULL) { \ - TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ - } \ - if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ - TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ - } \ - TD_DLIST_NELES(dl) -= 1; \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ - } while (0) - -#if 0 -// List iterator -#define TD_LIST_FITER 0 -#define TD_LIST_BITER 1 -#define TD_LIST_ITER(S) \ - struct { \ - int it_dir_; \ - S * it_next_; \ - S * it_ptr_; \ - TD_DLIST(S) * it_list_; \ - } - -#define tlistIterInit(it, l, dir) \ - (it)->it_dir_ = (dir); \ - (it)->it_list_ = l; \ - if ((dir) == TD_LIST_FITER) { \ - (it)->it_next_ = (l)->dl_head_; \ - } else { \ - (it)->it_next_ = (l)->dl_tail_; \ - } - -#define tlistIterNext(it) \ - ({ \ - (it)->it_ptr_ = (it)->it_next_; \ - if ((it)->it_next_ != NULL) { \ - if ((it)->it_dir_ == TD_LIST_FITER) { \ - (it)->it_next_ = (it)->it_next_->next_; \ - } else { \ - (it)->it_next_ = (it)->it_next_->prev_; \ - } \ - } \ - (it)->it_ptr_; \ - }) -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/include/util/tlist.h b/include/util/tlist.h index e803e96605..06d0abc797 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -19,19 +19,124 @@ extern "C" { #endif +// Single linked list ================ +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct TYPE *sl_next_; \ + } + +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ + } + +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) + +#define TD_SLIST_INIT(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ + } while (0) + +#define TD_SLIST_PUSH(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define TD_SLIST_POP(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + +// Double linked list ================ +#define TD_DLIST_NODE(TYPE) \ + struct { \ + struct TYPE *dl_prev_; \ + struct TYPE *dl_next_; \ + } + +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ + } + +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) + +#define TD_DLIST_INIT(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) + +#define TD_DLIST_APPEND(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define TD_DLIST_PREPEND(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define TD_DLIST_POP(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) + +// General double linked list typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; -typedef struct _list_node { - struct _list_node *next; - struct _list_node *prev; - char data[]; +typedef struct SListNode { + TD_DLIST_NODE(SListNode); + char data[]; } SListNode; typedef struct { - struct _list_node *head; - struct _list_node *tail; - int numOfEles; - int eleSize; + TD_DLIST(SListNode); + int eleSize; } SList; typedef struct { @@ -39,11 +144,11 @@ typedef struct { TD_LIST_DIRECTION_T direction; } SListIter; -#define listHead(l) (l)->head -#define listTail(l) (l)->tail -#define listNEles(l) (l)->numOfEles -#define listEleSize(l) (l)->eleSize -#define isListEmpty(l) ((l)->numOfEles == 0) +#define listHead(l) TD_DLIST_HEAD(l) +#define listTail(l) TD_DLIST_TAIL(l) +#define listNEles(l) TD_DLIST_NELES(l) +#define listEleSize(l) ((l)->eleSize) +#define isListEmpty(l) (TD_DLIST_NELES(l) == 0) #define listNodeFree(n) free(n) void tdListInit(SList *list, int eleSize); diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index b46afa31b9..c78bf02cbd 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -2,14 +2,13 @@ aux_source_directory(src CLIENT_SRC) add_library(taos ${CLIENT_SRC}) target_include_directories( taos - PUBLIC "${CMAKE_SOURCE_DIR}/include/client" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_SOURCE_DIR}/include/client" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( taos - PRIVATE common INTERFACE api - PRIVATE os util common transport parser + PRIVATE os util common transport parser catalog function qcom ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 3180923aff..4e7fff06b6 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -21,13 +21,14 @@ extern "C" { #endif #include "taos.h" +#include "common.h" #include "taosmsg.h" +#include "tdef.h" +#include "tep.h" #include "thash.h" #include "tlist.h" -#include "trpc.h" -#include "tdef.h" #include "tmsgtype.h" -#include "tep.h" +#include "trpc.h" typedef struct SQueryExecMetric { int64_t start; // start timestamp @@ -75,23 +76,39 @@ typedef struct SAppInfo { typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; - char acctId[TSDB_ACCT_ID_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + int32_t acctId; uint32_t connId; uint64_t id; // ref ID returned by taosAddRef -// struct SSqlObj *sqlList; void *pTransporter; pthread_mutex_t mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj from this tscObj SAppInstInfo *pAppInfo; } STscObj; +typedef struct SClientResultInfo { + const char *pMsg; + const char *pData; + TAOS_FIELD *fields; + int32_t numOfCols; + int32_t numOfRows; + int32_t current; + int32_t *length; + TAOS_ROW row; + char **pCol; +} SClientResultInfo; + typedef struct SReqBody { tsem_t rspSem; // not used now void* fp; void* param; + int32_t paramLen; + int64_t execId; // showId/queryId + SClientResultInfo* pResInfo; } SRequestBody; +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 + typedef struct SRequestObj { uint64_t requestId; int32_t type; // request type @@ -118,7 +135,7 @@ extern int32_t tscReqRef; extern void *tscQhandle; extern int32_t tscConnRef; -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); +extern int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody); extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); int taos_init(); @@ -129,8 +146,6 @@ void destroyTscObj(void*pObj); void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); - void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char *str); @@ -139,6 +154,12 @@ void* openTransporter(const char *user, const char *auth); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + +void* doFetchRow(SRequestObj* pRequest); +void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); + #ifdef __cplusplus } #endif diff --git a/source/client/inc/tscLog.h b/source/client/inc/clientLog.h similarity index 96% rename from source/client/inc/tscLog.h rename to source/client/inc/clientLog.h index f205a50227..bfa2c0319b 100644 --- a/source/client/inc/tscLog.h +++ b/source/client/inc/clientLog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCLOG_H -#define TDENGINE_TSCLOG_H +#ifndef TDENGINE_CLIENTLOG_H +#define TDENGINE_CLIENTLOG_H #ifdef __cplusplus extern "C" { diff --git a/source/client/src/client.c b/source/client/src/client.c index 863e96aa74..e69de29bb2 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tdef.h" -#include "tglobal.h" -#include "clientInt.h" -#include "tscLog.h" - -TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0)? port:tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } - - if (pass == NULL) { - pass = TSDB_DEFAULT_PASS; - } - - return taos_connect_internal(ip, user, pass, NULL, db, p); -} - -TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { - tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } - - if (auth == NULL) { - tscError("No auth info is given, failed to connect to server"); - return NULL; - } - - return taos_connect_internal(ip, user, NULL, auth, db, port); -} - -TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { - char ipStr[TSDB_EP_LEN] = {0}; - char dbStr[TSDB_DB_NAME_LEN] = {0}; - char userStr[TSDB_USER_LEN] = {0}; - char passStr[TSDB_PASSWORD_LEN] = {0}; - - strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); - strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipStr, userStr, passStr, dbStr, port); -} - diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2ae79f1947..b6200de824 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,15 +1,18 @@ -#include + #include "clientInt.h" +#include "clientLog.h" #include "tdef.h" #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" +#include "tnote.h" +#include "tpagedfile.h" #include "tref.h" -#include "tscLog.h" +#include "parser.h" static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); -static void destroyConnectMsg(SRequestMsgBody* pMsgBody); +static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody); static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); @@ -96,17 +99,84 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, char* key = getClusterKey(user, secretEncrypt, ip, port); - SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { - pInst = calloc(1, sizeof(struct SAppInstInfo)); + SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); - pInst->mgmtEp = epSet; - pInst->pTransporter = openTransporter(user, secretEncrypt); + p->mgmtEp = epSet; + p->pTransporter = openTransporter(user, secretEncrypt); + taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); - taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES); + pInst = &p; } - return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); + tfree(key); + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst); +} + +TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { + STscObj *pTscObj = (STscObj *)taos; + if (sqlLen > (size_t) tsMaxSQLStringLen) { + tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + nPrintTsc("%s", sql) + + SRequestObj* pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pRequest->sqlstr = malloc(sqlLen + 1); + if (pRequest->sqlstr == NULL) { + tscError("0x%"PRIx64" failed to prepare sql string buffer", pRequest->self); + + pRequest->msgBuf = strdup("failed to prepare sql string buffer"); + terrno = pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRequest; + } + + strntolower(pRequest->sqlstr, sql, (int32_t)sqlLen); + pRequest->sqlstr[sqlLen] = 0; + + tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr); + + int32_t code = 0; + if (qIsInsertSql(pRequest->sqlstr, sqlLen)) { + // todo add + } else { + int32_t type = 0; + void* output = NULL; + int32_t outputLen = 0; + code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { + pRequest->type = type; + pRequest->body.param = output; + pRequest->body.paramLen = outputLen; + + SRequestMsgBody body = {0}; + buildRequestMsgFp[type](pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyRequestMsgBody(&body); + } else { + assert(0); + } + } + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + return pRequest; + } + + return pRequest; } int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { @@ -166,7 +236,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); - destroyConnectMsg(&body); + destroyRequestMsgBody(&body); if (pRequest->code != TSDB_CODE_SUCCESS) { const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); @@ -213,7 +283,7 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) return 0; } -static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { +static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) { assert(pMsgBody != NULL); tfree(pMsgBody->pData); } @@ -269,11 +339,18 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); if (handleRequestRspFp[pRequest->type]) { - pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); + char *p = malloc(pMsg->contLen); + if (p == NULL) { + pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = pRequest->code; + } else { + memcpy(p, pMsg->pCont, pMsg->contLen); + pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, p, pMsg->contLen); + } } } else { - tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], - tstrerror(pMsg->code), pMsg->contLen); + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%"PRId64" ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); } taosReleaseRef(tscReqRef, requestRefId); @@ -281,3 +358,86 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { sem_post(&pRequest->body.rspSem); } + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} + +void* doFetchRow(SRequestObj* pRequest) { + assert(pRequest != NULL); + SClientResultInfo* pResultInfo = pRequest->body.pResInfo; + + if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { + pRequest->type = TSDB_SQL_RETRIEVE_MNODE; + + SRequestMsgBody body = {0}; + buildRequestMsgFp[pRequest->type](pRequest, &body); + + int64_t transporterId = 0; + STscObj* pTscObj = pRequest->pTscObj; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyRequestMsgBody(&body); + + pResultInfo->current = 0; + if (pResultInfo->numOfRows <= pResultInfo->current) { + return NULL; + } + } + + for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; + if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { + pResultInfo->length[i] = varDataLen(pResultInfo->row[i]); + pResultInfo->row[i] = varDataVal(pResultInfo->row[i]); + } + } + + pResultInfo->current += 1; + return pResultInfo->row; +} + +void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { + assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); + if (numOfRows == 0) { + return; + } + + int32_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pResultInfo->length[i] = pResultInfo->fields[i].bytes; + pResultInfo->row[i] = (char*) (pResultInfo->pData + offset * pResultInfo->numOfRows); + pResultInfo->pCol[i] = pResultInfo->row[i]; + offset += pResultInfo->fields[i].bytes; + } +} + +const char *taos_get_client_info() { return version; } + +int taos_affected_rows(TAOS_RES *res) { return 1; } + +int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c new file mode 100644 index 0000000000..21e632db8d --- /dev/null +++ b/source/client/src/clientMain.c @@ -0,0 +1,269 @@ +#include "clientInt.h" +#include "clientLog.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "trpc.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +static int32_t sentinel = TSC_VAR_NOT_RELEASE; + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + atomic_store_32(&lock, 0); + return ret; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + rpcCleanup(); + taosCloseLog(); +} + +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0) ? port : tsServerPort; + + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + +void taos_close(TAOS* taos) { + if (taos == NULL) { + return; + } + + STscObj *pTscObj = (STscObj *)taos; + tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + + taosRemoveRef(tscConnRef, pTscObj->id); +} + +int taos_errno(TAOS_RES *tres) { + if (tres == NULL) { + return terrno; + } + + return ((SRequestObj*) tres)->code; +} + +const char *taos_errstr(TAOS_RES *res) { + SRequestObj *pRequest = (SRequestObj *) res; + + if (pRequest == NULL) { + return (const char*) tstrerror(terrno); + } + + if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { + return pRequest->msgBuf; + } else { + return (const char*)tstrerror(pRequest->code); + } +} + +void taos_free_result(TAOS_RES *res) { + SRequestObj* pRequest = (SRequestObj*) res; + destroyRequest(pRequest); +} + +int taos_field_count(TAOS_RES *res) { + if (res == NULL) { + return 0; + } + + SRequestObj* pRequest = (SRequestObj*) res; + + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + if (pResInfo == NULL) { + return 0; + } + + return pResInfo->numOfCols; +} + +int taos_num_fields(TAOS_RES *res) { + return taos_field_count(res); +} + +TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { + if (taos_num_fields(res) == 0) { + return NULL; + } + + SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo; + return pResInfo->fields; +} + +TAOS_RES *taos_query(TAOS *taos, const char *sql) { + if (taos == NULL || sql == NULL) { + return NULL; + } + + return taos_query_l(taos, sql, strlen(sql)); +} + +TAOS_ROW taos_fetch_row(TAOS_RES *pRes) { + if (pRes == NULL) { + return NULL; + } + + SRequestObj *pRequest = (SRequestObj *) pRes; + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + pRequest->type == TSDB_SQL_INSERT) { + return NULL; + } + + return doFetchRow(pRequest); +} + +int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { + int32_t len = 0; + for (int i = 0; i < num_fields; ++i) { + if (i > 0) { + str[len++] = ' '; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); + if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + assert(charLen <= fields[i].bytes && charLen >= 0); + } else { + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + } + + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + + return len; +} + +int* taos_fetch_lengths(TAOS_RES *res) { + if (res == NULL) { + return NULL; + } + + return ((SRequestObj*) res)->body.pResInfo->length; +} + +const char *taos_data_type(int type) { + switch (type) { + case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; + case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL"; + case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT"; + case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT"; + case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT"; + case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT"; + case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT"; + case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE"; + case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY"; + case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP"; + case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR"; + default: return "UNKNOWN"; + } +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 62466a096d..36fa013f27 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,11 +13,13 @@ * along with this program. If not, see . */ -#include "os.h" +#include +#include #include "clientInt.h" +#include "clientLog.h" +#include "os.h" #include "tmsgtype.h" #include "trpc.h" -#include "tscLog.h" int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); @@ -528,7 +530,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) { if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || @@ -2699,7 +2701,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE) || + if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && @@ -2884,7 +2886,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg } int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { - assert(tIsValidName(&pTableMetaInfo->name)); + assert(tNameIsValid(&pTableMetaInfo->name)); char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); @@ -3091,6 +3093,36 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { #endif +int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + if (pConnect == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + // TODO refactor full_name + char *db; // ugly code to move the space + + STscObj *pObj = pRequest->pTscObj; + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + + pMsgBody->pData = pConnect; + return 0; +} + int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { STscObj *pTscObj = pRequest->pTscObj; @@ -3099,38 +3131,178 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pConnect->connId = htonl(pConnect->connId); pConnect->clusterId = htonl(pConnect->clusterId); + assert(pConnect->epSet.numOfEps > 0); + for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) { + pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]); + } + // TODO refactor pthread_mutex_lock(&pTscObj->mutex); char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; - int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); + int32_t len = sprintf(temp, "%d%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); assert(len <= sizeof(pTscObj->db)); tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); pthread_mutex_unlock(&pTscObj->mutex); - assert(pConnect->epSet.numOfEps > 0); if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); } for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id); } pTscObj->connId = pConnect->connId; + pTscObj->acctId = pConnect->acctId; // update the appInstInfo pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); - // createHbObj(pTscObj); + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); + pRequest->body.pResInfo->pMsg = pMsg; - // launch a timer to send heartbeat to maintain the connection and send status to mnode - // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); return 0; } +int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->requestObjRefId = pRequest->self; + pMsgBody->msgLen = pRequest->body.paramLen; + pMsgBody->pData = pRequest->body.param; + + switch(pRequest->type) { + case TSDB_SQL_CREATE_USER: + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; + break; + case TSDB_SQL_CREATE_DB: { + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; + + SCreateDbMsg* pCreateMsg = pRequest->body.param; + SName name = {0}; + int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db))); + if (ret != TSDB_CODE_SUCCESS) { + return -1; + } + + tNameGetFullDbName(&name, pCreateMsg->db); + + break; + } + case TSDB_SQL_SHOW: + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW; + break; + } +} + +STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); + + size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + + pTableMeta->tableType = pTableMetaMsg->tableType; + pTableMeta->vgId = pTableMetaMsg->vgId; + pTableMeta->suid = pTableMetaMsg->suid; + pTableMeta->uid = pTableMetaMsg->tuid; + + pTableMeta->tableInfo = (STableComInfo) { + .numOfTags = pTableMetaMsg->numOfTags, + .precision = pTableMetaMsg->precision, + .numOfColumns = pTableMetaMsg->numOfColumns, + }; + + pTableMeta->sversion = pTableMetaMsg->sversion; + pTableMeta->tversion = pTableMetaMsg->tversion; + + memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize); + + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < numOfTotalCols; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + SShowRsp* pShow = (SShowRsp *)pMsg; + pShow->showId = htonl(pShow->showId); + + STableMetaMsg *pMetaMsg = &(pShow->tableMeta); + pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); + + SSchema* pSchema = pMetaMsg->pSchema; + pMetaMsg->tuid = htobe64(pMetaMsg->tuid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htonl(pSchema->bytes); + pSchema++; + } + + pSchema = pMetaMsg->pSchema; + TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); + pFields[i].type = pSchema[i].type; + pFields[i].bytes = pSchema[i].bytes; + } + + if (pRequest->body.pResInfo == NULL) { + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); + } + + pRequest->body.pResInfo->pMsg = pMsg; + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + + pResInfo->fields = pFields; + pResInfo->numOfCols = pMetaMsg->numOfColumns; + pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + + pRequest->body.execId = pShow->showId; + return 0; +} + +int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pMsgBody->msgLen = sizeof(SRetrieveTableMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); + pRetrieveMsg->showId = htonl(pRequest->body.execId); + + pMsgBody->pData = pRetrieveMsg; + return TSDB_CODE_SUCCESS; +} + +int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + assert(msgLen >= sizeof(SRetrieveTableRsp)); + + tfree(pRequest->body.pResInfo->pMsg); + pRequest->body.pResInfo->pMsg = pMsg; + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg; + pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); + pRetrieve->precision = htons(pRetrieve->precision); + + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + pResInfo->numOfRows = pRetrieve->numOfRows; + pResInfo->pData = pRetrieve->data; // todo fix this in async model + + pResInfo->current = 0; + setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); + + tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, + pRetrieve->completed, pRequest->body.execId); + return 0; +} + +int32_t processCreateDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + // todo rsp with the vnode id list +} + void initMsgHandleFp() { #if 0 tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; @@ -3167,7 +3339,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -3185,7 +3357,7 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; @@ -3207,6 +3379,17 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; #endif -// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + buildRequestMsgFp[TSDB_SQL_CONNECT] = buildConnectMsg; handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; + + buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; + + buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg; + handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_DB] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_CREATE_DB] = processCreateDbRsp; } \ No newline at end of file diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c deleted file mode 100644 index ba80135850..0000000000 --- a/source/client/src/clientmain.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "clientInt.h" -#include "trpc.h" -#include "os.h" -#include "taosmsg.h" -#include "tcache.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "tref.h" -#include "tscLog.h" -#include "tsched.h" -#include "ttime.h" -#include "ttimezone.h" - -#define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 - -static int32_t sentinel = TSC_VAR_NOT_RELEASE; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; - -extern int32_t tscInitRes; - -int taos_options(TSDB_OPTION option, const void *arg, ...) { - static int32_t lock = 0; - - for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { - if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); - sched_yield(); - } - } - - int ret = taos_options_imp(option, (const char*)arg); - - atomic_store_32(&lock, 0); - return ret; -} - -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - -// this function may be called by user or system, or by both simultaneously. -void taos_cleanup(void) { - tscDebug("start to cleanup client environment"); - - if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { - return; - } - - int32_t id = tscReqRef; - tscReqRef = -1; - taosCloseRef(id); - - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); - - id = tscConnRef; - tscConnRef = -1; - taosCloseRef(id); - - rpcCleanup(); - taosCloseLog(); -} - -void taos_close(TAOS* taos) { - if (taos == NULL) { - return; - } - - STscObj *pTscObj = (STscObj *)taos; - tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); - - taosRemoveRef(tscConnRef, pTscObj->id); -} - -const char *taos_errstr(TAOS_RES *res) { - -} - -void taos_free_result(TAOS_RES *res) { - -} \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 43d73bf3db..023bd6ebe9 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include "clientInt.h" +#include "clientLog.h" #include "os.h" #include "taosmsg.h" #include "tcache.h" @@ -20,12 +22,10 @@ #include "tglobal.h" #include "tnote.h" #include "tref.h" -#include "tscLog.h" +#include "trpc.h" #include "tsched.h" #include "ttime.h" -#include "trpc.h" #include "ttimezone.h" -#include "clientInt.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -35,6 +35,7 @@ int32_t tscReqRef = -1; int32_t tscConnRef = -1; void *tscQhandle = NULL; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; int32_t tsNumOfThreads = 1; volatile int32_t tscInitRes = 0; @@ -64,7 +65,7 @@ static void deregisterRequest(SRequestObj* pRequest) { SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); taosReleaseRef(tscConnRef, pTscObj->id); @@ -126,8 +127,6 @@ void destroyTscObj(void *pObj) { atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - - closeTransporter(pTscObj); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } @@ -172,6 +171,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty pRequest->pTscObj = pObj; pRequest->body.fp = fp; pRequest->body.param = param; + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); @@ -188,6 +188,12 @@ static void doDestroyRequest(void* p) { tfree(pRequest->sqlstr); tfree(pRequest->pInfo); + if (pRequest->body.pResInfo != NULL) { + tfree(pRequest->body.pResInfo->pData); + tfree(pRequest->body.pResInfo->pMsg); + tfree(pRequest->body.pResInfo); + } + deregisterRequest(pRequest); tfree(pRequest); } @@ -250,6 +256,11 @@ void taos_init_imp(void) { tscDebug("client is initialized successfully"); } +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + int taos_options_imp(TSDB_OPTION option, const char *str) { SGlobalCfg *cfg = NULL; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8da4caf7eb..944247e88e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -14,6 +14,7 @@ */ #include +#include #include #include "tglobal.h" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -22,6 +23,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +#include "../inc/clientInt.h" #include "taos.h" namespace { @@ -33,8 +35,90 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { + taos_init(); +} + +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + taos_close(pConn); +} + +TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); taos_close(pConn); -} \ No newline at end of file +} + +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc"); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 28f920a6a9..fa303fe4e9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -4,9 +4,6 @@ #include "tname.h" #include "taosmsg.h" -#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) -#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) - #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) char* extractDBName(const char* tableId, char* name) { @@ -120,84 +117,11 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif -static struct SSchema _s = { - .colId = TSDB_TBNAME_COLUMN_INDEX, - .type = TSDB_DATA_TYPE_BINARY, - .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, - .name = "tbname", -}; - -SSchema* tGetTbnameColumnSchema() { - return &_s; -} - -static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { - int32_t rowLen = 0; - - for (int32_t i = 0; i < numOfCols; ++i) { - // 1. valid types - if (!isValidDataType(pSchema[i].type)) { - return false; - } - - // 2. valid length for each type - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { - if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { - return false; - } - } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { - return false; - } - } else { - if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { - return false; - } - } - - // 3. valid column names - for (int32_t j = i + 1; j < numOfCols; ++j) { - if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { - return false; - } - } - - rowLen += pSchema[i].bytes; - } - - return rowLen <= maxLen; -} - -bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { - if (!VALIDNUMOFCOLS(numOfCols)) { - return false; - } - - if (!VALIDNUMOFTAGS(numOfTags)) { - return false; - } - - /* first column must be the timestamp, which is a primary key */ - if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { - return false; - } - - if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { - return false; - } - - if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { - return false; - } - - return true; -} - int32_t tNameExtractFullName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); // invalid full name format, abort - if (!tIsValidName(name)) { + if (!tNameIsValid(name)) { return -1; } @@ -230,7 +154,7 @@ int32_t tNameLen(const SName* name) { } } -bool tIsValidName(const SName* name) { +bool tNameIsValid(const SName* name) { assert(name != NULL); if (!VALID_NAME_TYPE(name->type)) { @@ -265,13 +189,13 @@ int32_t tNameGetDbName(const SName* name, char* dst) { int32_t tNameGetFullDbName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, // there is a over write risk - "%s.%s", name->acctId, name->dbname); + "%d.%s", name->acctId, name->dbname); return 0; } bool tNameIsEmpty(const SName* name) { assert(name != NULL); - return name->type == 0 || strlen(name->acctId) <= 0; + return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { @@ -283,32 +207,23 @@ void tNameAssign(SName* dst, const SName* src) { memcpy(dst, src, sizeof(SName)); } -//int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) { -// assert(dst != NULL && dbToken != NULL && acct != NULL); -// -// // too long account id or too long db name -// if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) { -// return -1; -// } -// -// dst->type = TSDB_DB_NAME_T; -// tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); -// tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1); -// return 0; -//} - -int32_t tNameSetAcctId(SName* dst, const char* acct) { - assert(dst != NULL && acct != NULL); +int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) { + assert(dst != NULL && dbName != NULL && nameLen > 0); // too long account id or too long db name - if (strlen(acct) >= tListLen(dst->acctId)) { + if (nameLen >= tListLen(dst->dbname)) { return -1; } - tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + dst->type = TSDB_DB_NAME_T; + dst->acctId = acct; + tstrncpy(dst->dbname, dbName, nameLen + 1); + return 0; +} - assert(strlen(dst->acctId) > 0); - +int32_t tNameSetAcctId(SName* dst, int32_t acctId) { + assert(dst != NULL && acct != NULL); + dst->acctId = acctId; return 0; } @@ -325,14 +240,11 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { int32_t len = (int32_t)(p - str); // too long account id or too long db name - if ((len >= tListLen(dst->acctId)) || (len <= 0)) { - return -1; - } - - memcpy (dst->acctId, str, len); - dst->acctId[len] = 0; - - assert(strlen(dst->acctId) > 0); +// if ((len >= tListLen(dst->acctId)) || (len <= 0)) { +// return -1; +// } +// memcpy (dst->acctId, str, len); + dst->acctId = strtoll(str, NULL, 10); } if ((type & T_NAME_DB) == T_NAME_DB) { diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 0a725d9975..b9a1e4cecb 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -92,6 +92,7 @@ void dmnPrintVersion() { } int dmnReadConfig(const char *path) { + tstrncpy(configDir, global.configDir, PATH_MAX); taosInitGlobalCfg(); taosReadGlobalLogCfg(); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8202bfcf2a..50b1a1cf20 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -38,7 +38,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_TABLES_META] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index a6eb916aef..d3f1b06a4a 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -95,7 +95,7 @@ static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg); static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj * pVnode = NULL; + SVnodeObj *pVnode = NULL; int32_t refCount = 0; taosRLockLatch(&pMgmt->latch); @@ -107,23 +107,23 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { } taosRUnLockLatch(&pMgmt->latch); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + if (pVnode != NULL) { + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + } + return pVnode; } static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { + if (pVnode == NULL) return; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; - int32_t refCount = 0; taosRLockLatch(&pMgmt->latch); - if (pVnode != NULL) { - refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); - } + int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); taosRUnLockLatch(&pMgmt->latch); - if (pVnode != NULL) { - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); - } + dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) { @@ -457,7 +457,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { pMgmt->totalVnodes = numOfVnodes; - int32_t threadNum = tsNumOfCores; + int32_t threadNum = pDnode->opt.numOfCores; int32_t vnodesPerThread = numOfVnodes / threadNum + 1; SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); @@ -525,33 +525,49 @@ static void dndCloseVnodes(SDnode *pDnode) { static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { SCreateVnodeMsg *pCreate = rpcMsg->pCont; - *vgId = htonl(pCreate->vgId); + pCreate->vgId = htonl(pCreate->vgId); + pCreate->dnodeId = htonl(pCreate->dnodeId); + pCreate->dbUid = htobe64(pCreate->dbUid); + pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); + pCreate->daysPerFile = htonl(pCreate->daysPerFile); + pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); + pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); + pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); + pCreate->minRows = htonl(pCreate->minRows); + pCreate->maxRows = htonl(pCreate->maxRows); + pCreate->commitTime = htonl(pCreate->commitTime); + pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + for (int r = 0; r < pCreate->replica; ++r) { + SReplica *pReplica = &pCreate->replicas[r]; + pReplica->id = htonl(pReplica->id); + pReplica->port = htons(pReplica->port); + } + + *vgId = pCreate->vgId; #if 0 - tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN); - pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize); - pCfg->totalBlocks = htonl(pCreate->totalBlocks); - pCfg->daysPerFile = htonl(pCreate->daysPerFile); - pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0); - pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1); - pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); - pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); - pCfg->precision = pCreate->precision; - pCfg->compression = pCreate->compression; - pCfg->cacheLastRow = pCreate->cacheLastRow; - pCfg->update = pCreate->update; - pCfg->quorum = pCreate->quorum; - pCfg->replica = pCreate->replica; - pCfg->walLevel = pCreate->walLevel; - pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod); - - for (int32_t i = 0; i < pCfg->replica; ++i) { - pCfg->replicas[i].port = htons(pCreate->replicas[i].port); - tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); - } + pCfg->wsize = pCreate->cacheBlockSize; + pCfg->ssize = pCreate->cacheBlockSize; + pCfg->wsize = pCreate->cacheBlockSize; + pCfg->lsize = pCreate->cacheBlockSize; + pCfg->isHeapAllocator = true; + pCfg->ttl = 4; + pCfg->keep = pCreate->daysToKeep0; + pCfg->isWeak = true; + pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0; + pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; + pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; + pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; + pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; + pCfg->walCfg.level = pCreate->walLevel; + pCfg->walCfg.retentionPeriod = 10; + pCfg->walCfg.retentionSize = 128; + pCfg->walCfg.rollPeriod = 128; + pCfg->walCfg.segSize = 128; + pCfg->walCfg.vgId = pCreate->vgId; #endif - return 0; } @@ -1016,7 +1032,7 @@ static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { SVnodesMgmt * pMgmt = &pDnode->vmgmt; SMWorkerPool *pPool = &pMgmt->writePool; pPool->name = "vnode-write"; - pPool->max = tsNumOfCores; + pPool->max = pDnode->opt.numOfCores; if (tMWorkerInit(pPool) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1050,7 +1066,7 @@ static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { } static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) { - int32_t maxThreads = tsNumOfCores / 2; + int32_t maxThreads = pDnode->opt.numOfCores / 2; if (maxThreads < 1) maxThreads = 1; SVnodesMgmt *pMgmt = &pDnode->vmgmt; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 86186f77de..a5ece72f42 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,19 +1,20 @@ +enable_testing() add_subdirectory(acct) # add_subdirectory(auth) # add_subdirectory(balance) -# add_subdirectory(cluster) +add_subdirectory(cluster) add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) # add_subdirectory(mnode) # add_subdirectory(profile) # add_subdirectory(show) -# add_subdirectory(stb) +add_subdirectory(stb) # add_subdirectory(sync) # add_subdirectory(telem) # add_subdirectory(trans) add_subdirectory(user) -# add_subdirectory(vgroup) +add_subdirectory(vgroup) # add_subdirectory(common) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt index a548c2adc2..8b160fc446 100644 --- a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_acct "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_acct COMMAND dnode_test_acct diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index e1a71c5c9d..e2b819da28 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file acct.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module acct-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -49,7 +45,7 @@ SServer* DndTestAcct::pServer; SClient* DndTestAcct::pClient; int32_t DndTestAcct::connId; -TEST_F(DndTestAcct, CreateAcct) { +TEST_F(DndTestAcct, 01_CreateAcct) { ASSERT_NE(pClient, nullptr); SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(sizeof(SCreateAcctMsg)); @@ -65,7 +61,7 @@ TEST_F(DndTestAcct, CreateAcct) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } -TEST_F(DndTestAcct, AlterAcct) { +TEST_F(DndTestAcct, 02_AlterAcct) { ASSERT_NE(pClient, nullptr); SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(sizeof(SAlterAcctMsg)); @@ -81,7 +77,7 @@ TEST_F(DndTestAcct, AlterAcct) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } -TEST_F(DndTestAcct, DropAcct) { +TEST_F(DndTestAcct, 03_DropAcct) { ASSERT_NE(pClient, nullptr); SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(sizeof(SDropAcctMsg)); @@ -97,7 +93,7 @@ TEST_F(DndTestAcct, DropAcct) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } -TEST_F(DndTestAcct, ShowAcct) { +TEST_F(DndTestAcct, 04_ShowAcct) { ASSERT_NE(pClient, nullptr); SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt index b513f290e4..e97ef76881 100644 --- a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt @@ -1,29 +1,27 @@ -add_executable(dndTestCluster "") +add_executable(dnode_test_cluster "") -target_sources(dndTestCluster +target_sources(dnode_test_cluster PRIVATE "cluster.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestCluster + dnode_test_cluster PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestCluster +target_include_directories(dnode_test_cluster PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( - NAME dndTestCluster - COMMAND dndTestCluster + NAME dnode_test_cluster + COMMAND dnode_test_cluster ) diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index d47e63a85f..0aca2fde4b 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -1,170 +1,170 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file cluster.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module cluster-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" class DndTestCluster : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestCluster"; - const char* fqdn = "localhost"; - uint16_t port = 9521; + initLog("/tmp/tdlog"); - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9030"; + pServer = CreateServer("/tmp/dnode_test_cluster", fqdn, 9030, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9030); + taosMsleep(1100); } static void TearDownTestSuite() { stopServer(pServer); dropClient(pClient); + pServer = NULL; + pClient = NULL; } static SServer* pServer; static SClient* pClient; static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + if (db != NULL) { + strcpy(pShow->db, db); + } + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htonl(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt32() { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_GT(data, 0); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; }; SServer* DndTestCluster::pServer; SClient* DndTestCluster::pClient; int32_t DndTestCluster::connId; -TEST_F(DndTestCluster, ShowCluster) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; +TEST_F(DndTestCluster, 01_ShowCluster) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, "show cluster", 3, NULL); + CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "id"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_CLUSTER; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, "show cluster"); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 3); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "id"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "name"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "create_time"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - - char* pData = pRsp->data; - int32_t pos = 0; - - int32_t id = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - - int32_t nameLen = varDataLen(pData + pos); - pos += sizeof(VarDataLenT); - - char* name = (char*)(pData + pos); - pos += TSDB_CLUSTER_ID_LEN; - - int64_t create_time = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - - EXPECT_NE(id, 0); - EXPECT_EQ(nameLen, 36); - EXPECT_STRNE(name, ""); - EXPECT_GT(create_time, 0); - printf("--- id:%d nameLen:%d name:%s time:%" PRId64 " --- \n", id, nameLen, name, create_time); - } + SendThenCheckShowRetrieveMsg(1); + CheckInt32(); + CheckBinary(TSDB_CLUSTER_ID_LEN); + CheckTimestamp(); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt index b778e3854f..db96e2bad3 100644 --- a/source/dnode/mgmt/impl/test/db/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/db/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_db "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_db COMMAND dnode_test_db diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index b0ee00cb64..d465a62f2d 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file db.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module db-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -30,7 +26,7 @@ class DndTestDb : public ::testing::Test { const char* firstEp = "localhost:9040"; pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); pClient = createClient("root", "taosdata", fqdn, 9040); - taosMsleep(300); + taosMsleep(1100); } static void TearDownTestSuite() { @@ -48,11 +44,12 @@ class DndTestDb : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); pShow->type = showType; - strcpy(pShow->db, ""); - + if (db != NULL) { + strcpy(pShow->db, db); + } SRpcMsg showRpcMsg = {0}; showRpcMsg.pCont = pShow; showRpcMsg.contLen = sizeof(SShowMsg); @@ -67,10 +64,10 @@ class DndTestDb : public ::testing::Test { ASSERT_NE(pShowRsp, nullptr); pShowRsp->showId = htonl(pShowRsp->showId); pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); pMeta->tuid = htobe64(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); @@ -91,7 +88,7 @@ class DndTestDb : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -117,17 +114,14 @@ class DndTestDb : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -183,31 +177,198 @@ SClient* DndTestDb::pClient; int32_t DndTestDb::connId; TEST_F(DndTestDb, 01_ShowDb) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); - CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "days"); - CheckSchema(5, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); - CheckSchema(6, TSDB_DATA_TYPE_INT, 4, "cache(MB)"); - CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "blocks"); - CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "minrows"); - CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "maxrows"); - CheckSchema(10, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); - CheckSchema(11, TSDB_DATA_TYPE_INT, 4, "fsync"); - CheckSchema(12, TSDB_DATA_TYPE_TINYINT, 1, "comp"); - CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); - CheckSchema(14, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); - CheckSchema(15, TSDB_DATA_TYPE_TINYINT, 1, "update"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CheckSchema(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache"); + CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "blocks"); + CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "minrows"); + CheckSchema(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CheckSchema(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CheckSchema(12, TSDB_DATA_TYPE_INT, 4, "fsync"); + CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CheckSchema(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CheckSchema(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CheckSchema(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); SendThenCheckShowRetrieveMsg(0); } -TEST_F(DndTestDb, 02_CreateDb) { +TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { { SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); strcpy(pReq->db, "1.d1"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + // taosMsleep(1000000); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); // vgroups + CheckInt16(1); // replica + CheckInt16(1); // quorum + CheckInt16(10); // days + CheckBinary("3650,3650,3650", 24); // days + CheckInt32(16); // cache + CheckInt32(10); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(1); // wallevel + CheckInt32(3000); // fsync + CheckInt8(2); // comp + CheckInt8(0); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "show vgroups", 4, "1.d1"); + CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "vgId"); + CheckSchema(1, TSDB_DATA_TYPE_INT, 4, "tables"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); + CheckSchema(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); + SendThenCheckShowRetrieveMsg(2); + CheckInt32(1); + CheckInt32(2); + CheckInt32(0); + CheckInt32(0); + CheckInt16(1); + CheckInt16(1); + CheckBinary("master", 9); + CheckBinary("master", 9); + + { + SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->totalBlocks = htonl(12); + pReq->daysToKeep0 = htonl(300); + pReq->daysToKeep1 = htonl(400); + pReq->daysToKeep2 = htonl(500); + pReq->fsyncPeriod = htonl(4000); + pReq->walLevel = 2; + pReq->quorum = 2; + pReq->cacheLastRow = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); // vgroups + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update + + // restart + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9040"; + pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); // vgroups + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update + + { + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SDropDbMsg)); + strcpy(pReq->db, "1.d1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(0); +} + +TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { + { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.d2"); + pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); pReq->totalBlocks = htonl(10); pReq->daysPerFile = htonl(10); @@ -238,118 +399,64 @@ TEST_F(DndTestDb, 02_CreateDb) { ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); SendThenCheckShowRetrieveMsg(1); - CheckBinary("d1", TSDB_DB_NAME_LEN - 1); - CheckTimestamp(); - CheckInt16(1); // replica - CheckInt16(1); // quorum - CheckInt16(10); // days - CheckBinary("3650,3650,3650", 24); // days - CheckInt32(16); // cache - CheckInt32(10); // blocks - CheckInt32(100); // minrows - CheckInt32(4096); // maxrows - CheckInt8(1); // wallevel - CheckInt32(3000); // fsync - CheckInt8(2); // comp - CheckInt8(0); // cachelast - CheckBinary("ms", 3); // precision - CheckInt8(0); // update -} + CheckBinary("d2", TSDB_DB_NAME_LEN - 1); -TEST_F(DndTestDb, 03_AlterDb) { { - SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); - strcpy(pReq->db, "1.d1"); - pReq->totalBlocks = htonl(12); - pReq->daysToKeep0 = htonl(300); - pReq->daysToKeep1 = htonl(400); - pReq->daysToKeep2 = htonl(500); - pReq->fsyncPeriod = htonl(4000); - pReq->walLevel = 2; - pReq->quorum = 2; - pReq->cacheLastRow = 1; + SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg)); + strcpy(pReq->db, "1.d2"); + pReq->vgVersion = htonl(-1); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; + rpcMsg.contLen = sizeof(SUseDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB; sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); + + SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; + EXPECT_STREQ(pRsp->db, "1.d2"); + pRsp->vgVersion = htonl(pRsp->vgVersion); + pRsp->vgNum = htonl(pRsp->vgNum); + pRsp->hashMethod = pRsp->hashMethod; + EXPECT_EQ(pRsp->vgVersion, 1); + EXPECT_EQ(pRsp->vgNum, 2); + EXPECT_EQ(pRsp->hashMethod, 1); + + { + SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; + pInfo->vgId = htonl(pInfo->vgId); + pInfo->hashBegin = htonl(pInfo->hashBegin); + pInfo->hashEnd = htonl(pInfo->hashEnd); + EXPECT_GT(pInfo->vgId, 0); + EXPECT_EQ(pInfo->hashBegin, 0); + EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); + EXPECT_EQ(pInfo->inUse, 0); + EXPECT_EQ(pInfo->numOfEps, 1); + SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + pAddr->port = htons(pAddr->port); + EXPECT_EQ(pAddr->port, 9040); + EXPECT_STREQ(pAddr->fqdn, "localhost"); + } + + { + SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; + pInfo->vgId = htonl(pInfo->vgId); + pInfo->hashBegin = htonl(pInfo->hashBegin); + pInfo->hashEnd = htonl(pInfo->hashEnd); + EXPECT_GT(pInfo->vgId, 0); + EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); + EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); + EXPECT_EQ(pInfo->inUse, 0); + EXPECT_EQ(pInfo->numOfEps, 1); + SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + pAddr->port = htons(pAddr->port); + EXPECT_EQ(pAddr->port, 9040); + EXPECT_STREQ(pAddr->fqdn, "localhost"); + } } - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); - SendThenCheckShowRetrieveMsg(1); - CheckBinary("d1", TSDB_DB_NAME_LEN - 1); - CheckTimestamp(); - CheckInt16(1); // replica - CheckInt16(2); // quorum - CheckInt16(10); // days - CheckBinary("300,400,500", 24); // days - CheckInt32(16); // cache - CheckInt32(12); // blocks - CheckInt32(100); // minrows - CheckInt32(4096); // maxrows - CheckInt8(2); // wallevel - CheckInt32(4000); // fsync - CheckInt8(2); // comp - CheckInt8(1); // cachelast - CheckBinary("ms", 3); // precision - CheckInt8(0); // update } - -TEST_F(DndTestDb, 04_RestartDnode) { - stopServer(pServer); - pServer = NULL; - - uInfo("start all server"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9040"; - pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); - - uInfo("all server is running"); - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); - SendThenCheckShowRetrieveMsg(1); - CheckBinary("d1", TSDB_DB_NAME_LEN - 1); - CheckTimestamp(); - CheckInt16(1); // replica - CheckInt16(2); // quorum - CheckInt16(10); // days - CheckBinary("300,400,500", 24); // days - CheckInt32(16); // cache - CheckInt32(12); // blocks - CheckInt32(100); // minrows - CheckInt32(4096); // maxrows - CheckInt8(2); // wallevel - CheckInt32(4000); // fsync - CheckInt8(2); // comp - CheckInt8(1); // cachelast - CheckBinary("ms", 3); // precision - CheckInt8(0); // update -} - -TEST_F(DndTestDb, 05_DropDb) { - { - SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); - strcpy(pReq->db, "1.d1"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - } - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); - SendThenCheckShowRetrieveMsg(0); -} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt index ebe2f3c5eb..48b74a4eff 100644 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_dnode "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_dnode COMMAND dnode_test_dnode diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 5a4512e16c..1fa91f26cb 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -82,10 +78,10 @@ class DndTestDnode : public ::testing::Test { ASSERT_NE(pShowRsp, nullptr); pShowRsp->showId = htonl(pShowRsp->showId); pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); pMeta->tuid = htobe64(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); @@ -106,7 +102,7 @@ class DndTestDnode : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -132,17 +128,14 @@ class DndTestDnode : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -191,12 +184,12 @@ SClient* DndTestDnode::pClient; TEST_F(DndTestDnode, 01_ShowDnode) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max vnodes"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); - CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason"); + CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); SendThenCheckShowRetrieveMsg(1); CheckInt16(1); @@ -224,19 +217,21 @@ TEST_F(DndTestDnode, 02_ConfigDnode) { ASSERT_EQ(pMsg->code, 0); } -TEST_F(DndTestDnode, 03_CreateDnode) { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9042"); +TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9042"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); @@ -255,21 +250,21 @@ TEST_F(DndTestDnode, 03_CreateDnode) { CheckTimestamp(); CheckBinary("", 24); CheckBinary("", 24); -} -TEST_F(DndTestDnode, 04_DropDnode) { - SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); - pReq->dnodeId = htonl(2); + { + SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); + pReq->dnodeId = htonl(2); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendThenCheckShowRetrieveMsg(1); @@ -280,9 +275,7 @@ TEST_F(DndTestDnode, 04_DropDnode) { CheckBinary("ready", 10); CheckTimestamp(); CheckBinary("", 24); -} -TEST_F(DndTestDnode, 05_CreateDnode) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); strcpy(pReq->ep, "localhost:9043"); @@ -359,9 +352,8 @@ TEST_F(DndTestDnode, 05_CreateDnode) { CheckBinary("", 24); CheckBinary("", 24); CheckBinary("", 24); -} -TEST_F(DndTestDnode, 06_RestartDnode) { + // restart uInfo("stop all server"); stopServer(pServer1); stopServer(pServer2); diff --git a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt index 2e99a2e53d..d8e4eae042 100644 --- a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dndTestProfile "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dndTestProfile COMMAND dndTestProfile diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 2cbb9d5c2c..f1e4704777 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file profile.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module profile-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -220,17 +216,14 @@ TEST_F(DndTestProfile, SConnectMsg_03) { SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); pRsp->useconds = htobe64(pRsp->useconds); pRsp->compLen = htonl(pRsp->compLen); EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->offset, 0); EXPECT_EQ(pRsp->useconds, 0); EXPECT_EQ(pRsp->completed, 1); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); EXPECT_EQ(pRsp->compLen, 0); } } @@ -501,7 +494,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) { EXPECT_STREQ(pSchema->name, "queryId"); pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mgmt/impl/test/show/CMakeLists.txt index 5f1596ce95..b3dc3297a8 100644 --- a/source/dnode/mgmt/impl/test/show/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/show/CMakeLists.txt @@ -1,29 +1,27 @@ -add_executable(dndTestShow "") +add_executable(dnode_test_show "") -target_sources(dndTestShow +target_sources(dnode_test_show PRIVATE "show.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestShow + dnode_test_show PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestShow +target_include_directories(dnode_test_show PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( - NAME dndTestShow - COMMAND dndTestShow + NAME dnode_test_show + COMMAND dnode_test_show ) diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index c535386ecd..f0c20f71e5 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file show.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module show-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -155,49 +151,49 @@ TEST_F(DndTestShow, SShowMsg_04) { SSchema* pSchema = NULL; pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "program"); pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "pid"); pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "ip:port"); pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "login_time"); pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt new file mode 100644 index 0000000000..524e3e5c7e --- /dev/null +++ b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt @@ -0,0 +1,27 @@ +add_executable(dnode_test_stb "") + +target_sources(dnode_test_stb + PRIVATE + "stb.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dnode_test_stb + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dnode_test_stb + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +add_test( + NAME dnode_test_stb + COMMAND dnode_test_stb +) diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp new file mode 100644 index 0000000000..ee8c86bcea --- /dev/null +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -0,0 +1,374 @@ +/** + * @file stb.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module db-msg tests + * @version 0.1 + * @date 2021-12-17 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "deploy.h" + +class DndTestStb : public ::testing::Test { + protected: + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + initLog("/tmp/tdlog"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9101"; + pServer = CreateServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9101); + taosMsleep(1100); + } + + static void TearDownTestSuite() { + stopServer(pServer); + dropClient(pClient); + pServer = NULL; + pClient = NULL; + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + if (db != NULL) { + strcpy(pShow->db, db); + } + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htonl(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt8(int8_t val) { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + EXPECT_EQ(data, val); + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt32(int32_t val) { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +SServer* DndTestStb::pServer; +SClient* DndTestStb::pClient; +int32_t DndTestStb::connId; + +TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { + { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + int32_t cols = 2; + int32_t tags = 3; + int32_t size = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); + + SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(size); + strcpy(pReq->name, "1.d1.stb"); + pReq->numOfTags = htonl(tags); + pReq->numOfColumns = htonl(cols); + + { + SSchema* pSchema = &pReq->pSchema[0]; + pSchema->colId = htonl(0); + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema->name, "ts"); + } + + { + SSchema* pSchema = &pReq->pSchema[1]; + pSchema->colId = htonl(1); + pSchema->bytes = htonl(4); + pSchema->type = TSDB_DATA_TYPE_INT; + strcpy(pSchema->name, "col1"); + } + + { + SSchema* pSchema = &pReq->pSchema[2]; + pSchema->colId = htonl(2); + pSchema->bytes = htonl(2); + pSchema->type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema->name, "tag1"); + } + + { + SSchema* pSchema = &pReq->pSchema[3]; + pSchema->colId = htonl(3); + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema->name, "tag2"); + } + + { + SSchema* pSchema = &pReq->pSchema[4]; + pSchema->colId = htonl(4); + pSchema->bytes = htonl(16); + pSchema->type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema->name, "tag3"); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = size; + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_STB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CheckSchema(2, TSDB_DATA_TYPE_INT, 4, "columns"); + CheckSchema(3, TSDB_DATA_TYPE_INT, 4, "tags"); + + SendThenCheckShowRetrieveMsg(1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + + // ----- meta ------ + { + STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(sizeof(STableInfoMsg)); + strcpy(pReq->tableFname, "1.d1.stb"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(STableInfoMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_TABLE_META; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + STableMetaMsg* pRsp = (STableMetaMsg*)pMsg->pCont; + pRsp->numOfTags = htonl(pRsp->numOfTags); + pRsp->numOfColumns = htonl(pRsp->numOfColumns); + pRsp->sversion = htonl(pRsp->sversion); + pRsp->tversion = htonl(pRsp->tversion); + pRsp->suid = htobe64(pRsp->suid); + pRsp->tuid = htobe64(pRsp->tuid); + pRsp->vgId = htobe64(pRsp->vgId); + for (int32_t i = 0; i < pRsp->numOfTags + pRsp->numOfColumns; ++i) { + SSchema* pSchema = &pRsp->pSchema[i]; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + } + + EXPECT_STREQ(pRsp->tbFname, ""); + EXPECT_STREQ(pRsp->stbFname, "1.d1.stb"); + EXPECT_EQ(pRsp->numOfColumns, 2); + EXPECT_EQ(pRsp->numOfTags, 3); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->tableType, TSDB_SUPER_TABLE); + EXPECT_EQ(pRsp->update, 0); + EXPECT_EQ(pRsp->sversion, 1); + EXPECT_EQ(pRsp->tversion, 0); + EXPECT_GT(pRsp->suid, 0); + EXPECT_EQ(pRsp->tuid, 0); + EXPECT_EQ(pRsp->vgId, 0); + + { + SSchema* pSchema = &pRsp->pSchema[0]; + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "ts"); + } + } + + // restart + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9101"; + pServer = startServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckInt32(2); + CheckInt32(3); + + { + SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(sizeof(SDropStbMsg)); + strcpy(pReq->name, "1.d1.stb"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropStbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_STB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); + SendThenCheckShowRetrieveMsg(0); +} diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt index ca8f2ec6db..0d3215a4d0 100644 --- a/source/dnode/mgmt/impl/test/user/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_user "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_user COMMAND dnode_test_user diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 2b30dcbb1b..32309b106e 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file user.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module user-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -67,10 +63,10 @@ class DndTestUser : public ::testing::Test { ASSERT_NE(pShowRsp, nullptr); pShowRsp->showId = htonl(pShowRsp->showId); pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); pMeta->tuid = htobe64(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); @@ -91,7 +87,7 @@ class DndTestUser : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -117,17 +113,14 @@ class DndTestUser : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -174,7 +167,7 @@ TEST_F(DndTestUser, 01_ShowUser) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); SendThenCheckShowRetrieveMsg(1); @@ -184,7 +177,7 @@ TEST_F(DndTestUser, 01_ShowUser) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, 02_CreateUser) { +TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { { SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); strcpy(pReq->user, "u1"); @@ -231,23 +224,22 @@ TEST_F(DndTestUser, 02_CreateUser) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} -TEST_F(DndTestUser, 03_AlterUser) { - SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p2"); + { + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); SendThenCheckShowRetrieveMsg(3); CheckBinary("u1", TSDB_USER_LEN); @@ -262,22 +254,21 @@ TEST_F(DndTestUser, 03_AlterUser) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} -TEST_F(DndTestUser, 04_DropUser) { - SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); - strcpy(pReq->user, "u1"); + { + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + strcpy(pReq->user, "u1"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); SendThenCheckShowRetrieveMsg(2); CheckBinary("root", TSDB_USER_LEN); @@ -288,9 +279,8 @@ TEST_F(DndTestUser, 04_DropUser) { CheckTimestamp(); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} -TEST_F(DndTestUser, 05_RestartDnode) { + // restart stopServer(pServer); pServer = NULL; diff --git a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt new file mode 100644 index 0000000000..5670f9dbf2 --- /dev/null +++ b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt @@ -0,0 +1,27 @@ +add_executable(dnode_test_vgroup "") + +target_sources(dnode_test_vgroup + PRIVATE + "vgroup.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dnode_test_vgroup + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dnode_test_vgroup + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +add_test( + NAME dnode_test_vgroup + COMMAND dnode_test_vgroup +) diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp new file mode 100644 index 0000000000..3f16cd87d8 --- /dev/null +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -0,0 +1,224 @@ +/** + * @file db.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module vgroup-msg tests + * @version 0.1 + * @date 2021-12-20 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "deploy.h" + +class DndTestVgroup : public ::testing::Test { + protected: + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + initLog("/tmp/tdlog"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9150"; + pServer = CreateServer("/tmp/dnode_test_vgroup", fqdn, 9150, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9150); + taosMsleep(1100); + } + + static void TearDownTestSuite() { + stopServer(pServer); + dropClient(pClient); + pServer = NULL; + pClient = NULL; + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + if (db != NULL) { + strcpy(pShow->db, db); + } + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htonl(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt8(int8_t val) { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + EXPECT_EQ(data, val); + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt32(int32_t val) { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +SServer* DndTestVgroup::pServer; +SClient* DndTestVgroup::pClient; +int32_t DndTestVgroup::connId; + + +TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { + { + SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + taosMsleep(1000000); + } + +} + diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 115d896da2..36b6725737 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -42,13 +42,6 @@ extern int32_t mDebugFlag; #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} -typedef struct SClusterObj SClusterObj; -typedef struct SMnodeObj SMnodeObj; -typedef struct SAcctObj SAcctObj; -typedef struct SVgObj SVgObj; -typedef struct SFuncObj SFuncObj; -typedef struct SOperObj SOperObj; - typedef enum { MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_USER, @@ -99,7 +92,7 @@ typedef enum { DND_REASON_OTHERS } EDndReason; -typedef struct STrans { +typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; @@ -111,7 +104,7 @@ typedef struct STrans { SArray *undoActions; } STrans; -typedef struct SClusterObj { +typedef struct { int32_t id; char name[TSDB_CLUSTER_ID_LEN]; int64_t createdTime; @@ -139,7 +132,7 @@ typedef struct { char ep[TSDB_EP_LEN]; } SDnodeObj; -typedef struct SMnodeObj { +typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; @@ -167,7 +160,7 @@ typedef struct { int64_t compStorage; // Compressed storage on disk } SAcctInfo; -typedef struct SAcctObj { +typedef struct { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; @@ -189,14 +182,15 @@ typedef struct { } SUserObj; typedef struct { + int32_t numOfVgroups; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; + int32_t minRows; + int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; int8_t walLevel; @@ -214,7 +208,9 @@ typedef struct { int64_t createdTime; int64_t updateTime; int64_t uid; - int32_t version; + int32_t cfgVersion; + int32_t vgVersion; + int8_t hashMethod; // default is 1 SDbCfg cfg; } SDbObj; @@ -223,12 +219,15 @@ typedef struct { ESyncState role; } SVnodeGid; -typedef struct SVgObj { +typedef struct { int32_t vgId; int64_t createdTime; int64_t updateTime; int32_t version; + uint32_t hashBegin; + uint32_t hashEnd; char dbName[TSDB_FULL_DB_NAME_LEN]; + int64_t dbUid; int32_t numOfTables; int32_t numOfTimeSeries; int64_t totalStorage; @@ -245,6 +244,7 @@ typedef struct { int64_t createdTime; int64_t updateTime; uint64_t uid; + uint64_t dbUid; int32_t version; int32_t numOfColumns; int32_t numOfTags; @@ -252,7 +252,7 @@ typedef struct { SSchema *pSchema; } SStbObj; -typedef struct SFuncObj { +typedef struct { char name[TSDB_FUNC_NAME_LEN]; int64_t createdTime; int8_t funcType; diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 04ceb2820a..764dfbffc1 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,6 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); +bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 5da1d1ca2b..5c15e2f987 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -30,8 +30,8 @@ void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg); -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg); +int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); char *mndTransStageStr(ETrnStage stage); diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 21aee0f0a4..e9cdedd332 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -22,10 +22,15 @@ extern "C" { #endif -int32_t mndInitVgroup(SMnode *pMnode); -void mndCleanupVgroup(SMnode *pMnode); -SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); -void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); +int32_t mndInitVgroup(SMnode *pMnode); +void mndCleanupVgroup(SMnode *pMnode); +SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); +void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); + +SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); +SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 0eaa184907..7c53a4ebd0 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -143,22 +143,22 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); strcpy(pMeta->tbFname, mndShowStr(pShow->type)); pShow->numOfColumns = cols; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9f2b2d17f3..e85edf66da 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -20,6 +20,7 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" +#include "mndVgroup.h" #define TSDB_DB_VER_NUM 1 #define TSDB_DB_RESERVE_SIZE 64 @@ -74,15 +75,18 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) SDB_SET_INT64(pRaw, dataPos, pDb->uid) - SDB_SET_INT32(pRaw, dataPos, pDb->version) + SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion) + SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion) + SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRowsPerFileBlock) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRowsPerFileBlock) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel) @@ -118,15 +122,18 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfgVersion) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->vgVersion) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->hashMethod) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.numOfVgroups) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep0) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep1) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep2) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRowsPerFileBlock) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRowsPerFileBlock) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRows) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRows) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.commitTime) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.fsyncPeriod) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel) @@ -154,6 +161,8 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); pOldDb->updateTime = pNewDb->createdTime; + pOldDb->cfgVersion = pNewDb->cfgVersion; + pOldDb->vgVersion = pNewDb->vgVersion; memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); return 0; } @@ -184,145 +193,44 @@ static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) { return 0; } -static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg, char *errMsg, int32_t len) { - if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database cache block size option", len); - return -1; - } - - if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database total blocks option", len); - return -1; - } - - if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database days option", len); - return -1; - } - - if (pCfg->daysToKeep0 < pCfg->daysPerFile) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database days option", len); - return -1; - } - - if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database keep0 option", len); - return -1; - } - - if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database keep1 option", len); - return -1; - } - - if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database keep2 option", len); - return -1; - } - - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database minrows option", len); - return -1; - } - - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database maxrows option", len); - return -1; - } - - if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database minrows option", len); - return -1; - } - - if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database commit option", len); - return -1; - } - - if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database fsync option", len); - return -1; - } - - if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database wal level option", len); - return -1; - } - - if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid precision option", len); - return -1; - } - - if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database compression option", len); - return -1; - } - - if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database replication option", len); - return -1; - } - - if (pCfg->replications > mndGetDnodeSize(pMnode)) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database replication option", len); - return -1; - } - - if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database quorum option", len); - return -1; - } - - if (pCfg->quorum > pCfg->replications) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database quorum option", len); - return -1; - } - - if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database update option", len); - return -1; - } - - if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database cachelast option", len); - return -1; - } - +static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; + if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; + if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1; + if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1; + if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1; + if (pCfg->maxRows < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRows > TSDB_MAX_MAX_ROW_FBLOCK) return -1; + if (pCfg->minRows > pCfg->maxRows) return -1; + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1; + if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; + if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; + if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; + if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) return -1; + if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; + if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) return -1; + if (pCfg->quorum > pCfg->replications) return -1; + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1; + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; return TSDB_CODE_SUCCESS; } static void mndSetDefaultDbCfg(SDbCfg *pCfg) { + if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = TSDB_DEFAULT_KEEP; - if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; - if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MIN_ROW_FBLOCK; + if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK; if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; @@ -334,6 +242,106 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } +static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + SSdbRaw *pDbRaw = mndDbActionEncode(pDb); + if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; + sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); + + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); + if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); + } + + return 0; +} + +static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + SSdbRaw *pDbRaw = mndDbActionEncode(pDb); + if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; + sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); + + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); + if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; + sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); + } + + return 0; +} + +static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + SSdbRaw *pDbRaw = mndDbActionEncode(pDb); + if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; + sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); + + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); + if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + return 0; +} + +static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + SVgObj *pVgroup = pVgroups + v; + + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) { + return -1; + } + + SEpSet epset = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); + if (pMsg == NULL) { + return -1; + } + + if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) { + free(pMsg); + return -1; + } + } + } + + return 0; +} + +static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + SVgObj *pVgroup = pVgroups + v; + + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) { + return -1; + } + + SEpSet epset = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); + if (pMsg == NULL) { + return -1; + } + + if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) { + free(pMsg); + return -1; + } + } + } + + return 0; +} + static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); @@ -341,14 +349,18 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); - dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, + dbObj.hashMethod = 1; + dbObj.cfgVersion = 1; + dbObj.vgVersion = 1; + dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups, + .cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, .daysPerFile = pCreate->daysPerFile, .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRowsPerFileBlock = pCreate->minRowsPerFileBlock, - .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .minRows = pCreate->minRowsPerFileBlock, + .maxRows = pCreate->maxRowsPerFileBlock, .fsyncPeriod = pCreate->fsyncPeriod, .commitTime = pCreate->commitTime, .precision = pCreate->precision, @@ -366,57 +378,69 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat return -1; } - char errMsg[TSDB_ERROR_MSG_LEN] = {0}; - if (mndCheckDbCfg(pMnode, &dbObj.cfg, errMsg, TSDB_ERROR_MSG_LEN) != 0) { + if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } + SVgObj *pVgroups = NULL; + if (mndAllocVgroup(pMnode, &dbObj, &pVgroups) != 0) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - SSdbRaw *pRedoRaw = mndDbActionEncode(&dbObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - SSdbRaw *pUndoRaw = mndDbActionEncode(&dbObj); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; + } + + if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; + } + + if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto CREATE_DB_OVER; } + code = 0; + +CREATE_DB_OVER: + free(pVgroups); mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; + pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); @@ -573,7 +597,8 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return code; } - dbObj.version++; + dbObj.cfgVersion++; + dbObj.updateTime = taosGetTimestampMs(); code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); mndReleaseDb(pMnode, pDb); @@ -658,17 +683,69 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; SUseDbMsg *pUse = pMsg->rpcMsg.pCont; + pUse->vgVersion = htonl(pUse->vgVersion); - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); - if (pDb != NULL) { - strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); - mndReleaseDb(pMnode, pDb); - return 0; - } else { - mError("db:%s, failed to process use db msg since %s", pMsg->db, terrstr()); + SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mError("db:%s, failed to process use db msg since %s", pUse->db, terrstr()); return -1; } + + int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); + SUseDbRsp *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t vindex = 0; + + if (pUse->vgVersion < pDb->vgVersion) { + void *pIter = NULL; + while (vindex < pDb->cfg.numOfVgroups) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pDb->uid) { + SVgroupInfo *pInfo = &pRsp->vgroupInfo[vindex]; + pInfo->vgId = htonl(pVgroup->vgId); + pInfo->hashBegin = htonl(pVgroup->hashBegin); + pInfo->hashEnd = htonl(pVgroup->hashEnd); + pInfo->numOfEps = pVgroup->replica; + for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; + SEpAddrMsg *pEpArrr = &pInfo->epAddr[gid]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode != NULL) { + memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + pEpArrr->port = htons(pDnode->port); + } + mndReleaseDnode(pMnode, pDnode); + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + pInfo->inUse = gid; + } + } + vindex++; + } + + sdbRelease(pSdb, pVgroup); + } + } + + memcpy(pRsp->db, pDb->name, TSDB_FULL_DB_NAME_LEN); + pRsp->vgVersion = htonl(pDb->vgVersion); + pRsp->vgNum = htonl(vindex); + pRsp->hashMethod = pDb->hashMethod; + + pMsg->pCont = pRsp; + pMsg->contLen = contLen; + mndReleaseDb(pMnode, pDb); + + return 0; } static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { @@ -709,100 +786,106 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "vgroups"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "replica"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "quorum"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "days"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "keep0,keep1,keep2"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(MB)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "cache"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "blocks"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "minrows"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "maxrows"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "wallevel"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "fsync"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "comp"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "cachelast"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "precision"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "update"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -855,6 +938,10 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 *(int64_t *)pWrite = pDb->createdTime; cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.numOfVgroups; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int16_t *)pWrite = pDb->cfg.replications; cols++; @@ -886,11 +973,11 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; + *(int32_t *)pWrite = pDb->cfg.minRows; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; + *(int32_t *)pWrite = pDb->cfg.maxRows; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ec5f68a713..493f20bc9a 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -180,8 +180,12 @@ static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj } SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_DNODE, &dnodeId); + SSdb *pSdb = pMnode->pSdb; + SDnodeObj *pDnode = sdbAcquire(pSdb, SDB_DNODE, &dnodeId); + if (pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + } + return pDnode; } void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { @@ -218,6 +222,15 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } +bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { + int64_t ms = taosGetTimestampMs(); + int64_t interval = ABS(pDnode->lastAccessTime - ms); + if (interval > 3000 * pMnode->cfg.statusInterval) { + return false; + } + return true; +} + static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { SSdb *pSdb = pMnode->pSdb; @@ -552,16 +565,16 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -629,46 +642,46 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "max vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "max_vnodes"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "offline reason"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "offline_reason"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b7158bb094..a513c9d57d 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -380,46 +380,46 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "comment"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "aggregate"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "outputtype"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "code_len"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "bufsize"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 7b6804d43e..16733cbf94 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -368,34 +368,34 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "role time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "role_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 45a63f2dc5..bf657fd27a 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -474,48 +474,48 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; // app name pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "program"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; // app pid pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "login_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "last_access"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -602,88 +602,88 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "queryId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "qid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "created_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql_obj_id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ep"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_BOOL; strcpy(pSchema[cols].name, "stable_query"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "sub_queries"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sub_query_info"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -818,64 +818,64 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "streamId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "dest table"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "destination"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "exec time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "exec"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time(us)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "cycles"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 91a229956f..db6777ebf8 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -291,7 +291,7 @@ char *mndShowStr(int32_t showType) { case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; case TSDB_MGMT_TABLE_CLUSTER: - return "show clusters"; + return "show cluster"; case TSDB_MGMT_TABLE_STREAMTABLES: return "show streamtables"; case TSDB_MGMT_TABLE_TP: diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 791b6f5d12..822036b599 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -74,11 +74,13 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { if (pRaw == NULL) return NULL; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN) + SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN) SDB_SET_INT64(pRaw, dataPos, pStb->createdTime) SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) SDB_SET_INT64(pRaw, dataPos, pStb->uid) - SDB_SET_INT64(pRaw, dataPos, pStb->version) + SDB_SET_INT64(pRaw, dataPos, pStb->dbUid) + SDB_SET_INT32(pRaw, dataPos, pStb->version) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) @@ -113,10 +115,12 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { if (pStb == NULL) return NULL; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->dbUid) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) @@ -153,11 +157,17 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb atomic_exchange_32(&pOldStb->version, pNewStb->version); taosWLockLatch(&pOldStb->lock); + pOldStb->numOfColumns = pNewStb->numOfColumns; + pOldStb->numOfTags = pNewStb->numOfTags; int32_t totalCols = pNewStb->numOfTags + pNewStb->numOfColumns; int32_t totalSize = totalCols * sizeof(SSchema); if (pOldStb->numOfTags + pOldStb->numOfColumns < totalCols) { - pOldStb->pSchema = malloc(totalSize); + void *pSchema = malloc(totalSize); + if (pSchema != NULL) { + free(pOldStb->pSchema); + pOldStb->pSchema = pSchema; + } } memcpy(pOldStb->pSchema, pNewStb->pSchema, totalSize); @@ -196,37 +206,37 @@ static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { } if (pCreate->igExists < 0 || pCreate->igExists > 1) { - terrno = TSDB_CODE_MND_STB_INVALID_IGEXIST; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pCreate->numOfColumns < TSDB_MIN_COLUMNS || pCreate->numOfColumns > TSDB_MAX_COLUMNS) { - terrno = TSDB_CODE_MND_STB_INVALID_COLS_NUM; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pCreate->numOfTags <= 0 || pCreate->numOfTags > TSDB_MAX_TAGS) { - terrno = TSDB_CODE_MND_STB_INVALID_TAGS_NUM; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } int32_t maxColId = (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS); for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pCreate->pSchema[i]; - if (pSchema->type <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + if (pSchema->type < 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->colId < 0 || pSchema->colId >= maxColId) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->bytes <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->name[0] == 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } } @@ -241,6 +251,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre stbObj.createdTime = taosGetTimestampMs(); stbObj.updateTime = stbObj.createdTime; stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + stbObj.dbUid = pDb->uid; stbObj.version = 1; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; @@ -346,19 +357,19 @@ static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { pSchema->bytes = htonl(pSchema->bytes); if (pSchema->type <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->bytes <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->name[0] == 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -476,31 +487,37 @@ static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SStbInfoMsg *pInfo = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - mDebug("stb:%s, start to retrieve meta", pInfo->name); + mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); - SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->name); + SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->tableFname); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); return -1; } - SStbObj *pStb = mndAcquireStb(pMnode, pInfo->name); + SStbObj *pStb = mndAcquireStb(pMnode, pInfo->tableFname); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; - mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); + terrno = TSDB_CODE_MND_INVALID_STB; + mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); return -1; } - int32_t contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + taosRLockLatch(&pStb->lock); + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + STableMetaMsg *pMeta = rpcMallocCont(contLen); if (pMeta == NULL) { + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); return -1; } @@ -513,7 +530,7 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { pMeta->sversion = htonl(pStb->version); pMeta->suid = htonl(pStb->uid); - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pMeta->pSchema[i]; SSchema *pSrcSchema = &pStb->pSchema[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); @@ -521,11 +538,14 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { pSchema->colId = htonl(pSrcSchema->colId); pSchema->bytes = htonl(pSrcSchema->bytes); } + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); pMsg->pCont = pMeta; pMsg->contLen = contLen; - mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pStb->numOfColumns, pStb->numOfTags); return 0; } @@ -542,7 +562,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs void *pIter = NULL; while (1) { SStbObj *pStb = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); if (pIter == NULL) break; if (strcmp(pStb->db, dbName) == 0) { @@ -570,28 +590,28 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tags"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -599,6 +619,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } + pShow->numOfRows = sdbGetSize(pSdb, SDB_STB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; strcpy(pMeta->tbFname, mndShowStr(pShow->type)); @@ -642,8 +663,8 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols = 0; - char stbName[TSDB_TABLE_FNAME_LEN] = {0}; - memcpy(stbName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + char stbName[TSDB_TABLE_NAME_LEN] = {0}; + tstrncpy(stbName, pStb->name + prefixLen, TSDB_TABLE_NAME_LEN); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, stbName); cols++; @@ -653,11 +674,11 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStb->numOfColumns; + *(int32_t *)pWrite = pStb->numOfColumns; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStb->numOfTags; + *(int32_t *)pWrite = pStb->numOfTags; cols++; numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 32ac795301..5caec6c78d 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -21,6 +21,13 @@ #define TSDB_TRN_ARRAY_SIZE 8 #define TSDB_TRN_RESERVE_SIZE 64 +typedef struct { + SEpSet epSet; + int8_t msgType; + int32_t contLen; + void *pCont; +} STransAction; + static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); @@ -29,9 +36,12 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); -static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw); -static void mndTransDropArray(SArray *pArray); -static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray); +static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); +static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +static void mndTransDropLogs(SArray *pArray); +static void mndTransDropActions(SArray *pArray); +static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray); +static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); @@ -58,7 +68,7 @@ int32_t mndInitTrans(SMnode *pMnode) { void mndCleanupTrans(SMnode *pMnode) {} static SSdbRaw *mndTransActionEncode(STrans *pTrans) { - int32_t rawDataLen = 16 * sizeof(int32_t) + TSDB_TRN_RESERVE_SIZE; + int32_t rawDataLen = sizeof(STrans) + TSDB_TRN_RESERVE_SIZE; int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -80,6 +90,16 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += sdbGetRawTotalSize(pTmp); } + for (int32_t i = 0; i < redoActionNum; ++i) { + STransAction *pAction = taosArrayGet(pTrans->redoActions, i); + rawDataLen += (sizeof(STransAction) + pAction->contLen); + } + + for (int32_t i = 0; i < undoActionNum; ++i) { + STransAction *pAction = taosArrayGet(pTrans->undoActions, i); + rawDataLen += (sizeof(STransAction) + pAction->contLen); + } + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); @@ -116,6 +136,22 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } + for (int32_t i = 0; i < redoActionNum; ++i) { + STransAction *pAction = taosArrayGet(pTrans->redoActions, i); + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet)); + SDB_SET_INT8(pRaw, dataPos, pAction->msgType) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen) + SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen); + } + + for (int32_t i = 0; i < undoActionNum; ++i) { + STransAction *pAction = taosArrayGet(pTrans->undoActions, i); + SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet)); + SDB_SET_INT8(pRaw, dataPos, pAction->msgType) + SDB_SET_INT32(pRaw, dataPos, pAction->contLen) + SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen); + } + SDB_SET_RESERVE(pRaw, dataPos, TSDB_TRN_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos); @@ -147,8 +183,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -175,42 +211,77 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < redoLogNum; ++i) { int32_t dataLen = 0; SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) - char *pData = malloc(dataLen); SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->redoLogs, &pData); if (ret == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto TRANS_DECODE_OVER; - break; } } for (int32_t i = 0; i < undoLogNum; ++i) { int32_t dataLen = 0; SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) - char *pData = malloc(dataLen); SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->undoLogs, &pData); if (ret == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto TRANS_DECODE_OVER; - break; } } for (int32_t i = 0; i < commitLogNum; ++i) { int32_t dataLen = 0; SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) - char *pData = malloc(dataLen); SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->commitLogs, &pData); if (ret == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto TRANS_DECODE_OVER; - break; + } + } + + for (int32_t i = 0; i < redoActionNum; ++i) { + STransAction action = {0}; + SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet)); + SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType) + SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen) + action.pCont = malloc(action.contLen); + if (action.pCont == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + } + SDB_GET_BINARY(pRaw, pRow, dataPos, action.pCont, action.contLen); + + void *ret = taosArrayPush(pTrans->redoActions, &action); + if (ret == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + } + } + + for (int32_t i = 0; i < undoActionNum; ++i) { + STransAction action = {0}; + SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet)); + SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType) + SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen) + action.pCont = malloc(action.contLen); + if (action.pCont == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + } + SDB_GET_BINARY(pRaw, pRow, dataPos, action.pCont, action.contLen); + + void *ret = taosArrayPush(pTrans->undoActions, &action); + if (ret == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; } } @@ -237,11 +308,11 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { mTrace("trans:%d, perform delete action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage)); - mndTransDropArray(pTrans->redoLogs); - mndTransDropArray(pTrans->undoLogs); - mndTransDropArray(pTrans->commitLogs); - mndTransDropArray(pTrans->redoActions); - mndTransDropArray(pTrans->undoActions); + mndTransDropLogs(pTrans->redoLogs); + mndTransDropLogs(pTrans->undoLogs); + mndTransDropLogs(pTrans->commitLogs); + mndTransDropActions(pTrans->redoActions); + mndTransDropActions(pTrans->undoActions); return 0; } @@ -274,6 +345,8 @@ char *mndTransStageStr(ETrnStage stage) { return "rollback"; case TRN_STAGE_RETRY: return "retry"; + case TRN_STAGE_OVER: + return "stop"; default: return "undefined"; } @@ -305,8 +378,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); - pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *)); + pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); + pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction)); if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { @@ -319,7 +392,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return pTrans; } -static void mndTransDropArray(SArray *pArray) { +static void mndTransDropLogs(SArray *pArray) { for (int32_t i = 0; i < pArray->size; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); tfree(pRaw); @@ -328,12 +401,21 @@ static void mndTransDropArray(SArray *pArray) { taosArrayDestroy(pArray); } +static void mndTransDropActions(SArray *pArray) { + for (int32_t i = 0; i < pArray->size; ++i) { + STransAction *pAction = taosArrayGet(pArray, i); + free(pAction->pCont); + } + + taosArrayDestroy(pArray); +} + void mndTransDrop(STrans *pTrans) { - mndTransDropArray(pTrans->redoLogs); - mndTransDropArray(pTrans->undoLogs); - mndTransDropArray(pTrans->commitLogs); - mndTransDropArray(pTrans->redoActions); - mndTransDropArray(pTrans->undoActions); + mndTransDropLogs(pTrans->redoLogs); + mndTransDropLogs(pTrans->undoLogs); + mndTransDropLogs(pTrans->commitLogs); + mndTransDropActions(pTrans->redoActions); + mndTransDropActions(pTrans->undoActions); mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); tfree(pTrans); @@ -344,7 +426,7 @@ static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle); } -static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { +static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { if (pArray == NULL || pRaw == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -360,32 +442,44 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { } int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { - int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw); + int32_t code = mndTransAppendLog(pTrans->redoLogs, pRaw); mTrace("trans:%d, raw:%p append to redo logs, code:0x%x", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { - int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw); + int32_t code = mndTransAppendLog(pTrans->undoLogs, pRaw); mTrace("trans:%d, raw:%p append to undo logs, code:0x%x", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { - int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw); + int32_t code = mndTransAppendLog(pTrans->commitLogs, pRaw); mTrace("trans:%d, raw:%p append to commit logs, code:0x%x", pTrans->id, pRaw, code); return code; } -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { - int32_t code = mndTransAppendArray(pTrans->redoActions, pMsg); - mTrace("trans:%d, msg:%p append to redo actions", pTrans->id, pMsg); +static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { + STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont}; + + void *ptr = taosArrayPush(pArray, &action); + if (ptr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { + int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont); + mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen); return code; } -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { - int32_t code = mndTransAppendArray(pTrans->undoActions, pMsg); - mTrace("trans:%d, msg:%p append to undo actions", pTrans->id, pMsg); +int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { + int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont); + mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen); return code; } @@ -502,7 +596,7 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) // todo } -static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) { +static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { SSdb *pSdb = pMnode->pSdb; int32_t arraySize = taosArrayGetSize(pArray); @@ -520,7 +614,7 @@ static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) { static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; if (taosArrayGetSize(pTrans->redoLogs) != 0) { - code = mndTransExecuteArray(pMnode, pTrans->redoLogs); + code = mndTransExecuteLogs(pMnode, pTrans->redoLogs); if (code != 0) { mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) } else { @@ -534,7 +628,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; if (taosArrayGetSize(pTrans->undoLogs) != 0) { - code = mndTransExecuteArray(pMnode, pTrans->undoLogs); + code = mndTransExecuteLogs(pMnode, pTrans->undoLogs); if (code != 0) { mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) } else { @@ -548,7 +642,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; if (taosArrayGetSize(pTrans->commitLogs) != 0) { - code = mndTransExecuteArray(pMnode, pTrans->commitLogs); + code = mndTransExecuteLogs(pMnode, pTrans->commitLogs); if (code != 0) { mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) } else { @@ -559,18 +653,40 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { return code; } -static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->redoActions) != 0) { - mTrace("trans:%d, execute redo actions finished", pTrans->id); +static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) { +#if 0 + int32_t arraySize = taosArrayGetSize(pArray); + for (int32_t i = 0; i < arraySize; ++i) { + STransAction *pAction = taosArrayGet(pArray, i); + + SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen}; + rpcMsg.pCont = rpcMallocCont(pAction->contLen); + if (rpcMsg.pCont == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); + mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg); } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +#else return 0; +#endif +} + +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { + if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0; + + mTrace("trans:%d, start to execute redo actions", pTrans->id); + return mndTransExecuteActions(pMnode, pTrans->redoActions); } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->undoActions) != 0) { - mTrace("trans:%d, execute undo actions finished", pTrans->id); - } - return 0; + if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0; + + mTrace("trans:%d, start to execute undo actions", pTrans->id); + return mndTransExecuteActions(pMnode, pTrans->undoActions); } static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7219342d59..58d687a148 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -410,28 +410,28 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "privilege"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "account"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 9a462c024b..e99fea200b 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -24,7 +24,6 @@ #define TSDB_VGROUP_VER_NUM 1 #define TSDB_VGROUP_RESERVE_SIZE 64 -static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); @@ -70,8 +69,8 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} -static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); +SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -79,7 +78,10 @@ static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime) SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime) SDB_SET_INT32(pRaw, dataPos, pVgroup->version) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd) SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid) SDB_SET_INT8(pRaw, dataPos, pVgroup->replica) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; @@ -92,7 +94,7 @@ static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { return pRaw; } -static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { +SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; @@ -102,7 +104,7 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { return NULL; } - SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); + SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); SVgObj *pVgroup = sdbGetRowObj(pRow); if (pVgroup == NULL) return NULL; @@ -111,7 +113,10 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->updateTime) SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashBegin) + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashEnd) SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->dbUid) SDB_GET_INT8(pRaw, pRow, dataPos, &pVgroup->replica) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; @@ -133,12 +138,6 @@ static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) { return 0; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } - static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { mTrace("vgId:%d, perform update action", pOldVgroup->vgId); pOldVgroup->updateTime = pNewVgroup->updateTime; @@ -158,6 +157,160 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } +SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { + SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg)); + if (pCreate == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCreate->dnodeId = htonl(pDnode->id); + pCreate->vgId = htonl(pVgroup->vgId); + memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN); + pCreate->dbUid = htobe64(pDb->uid); + pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); + pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks); + pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile); + pCreate->daysToKeep0 = htonl(pDb->cfg.daysToKeep0); + pCreate->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); + pCreate->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); + pCreate->minRows = htonl(pDb->cfg.minRows); + pCreate->maxRows = htonl(pDb->cfg.maxRows); + pCreate->commitTime = htonl(pDb->cfg.commitTime); + pCreate->fsyncPeriod = htonl(pDb->cfg.fsyncPeriod); + pCreate->walLevel = pDb->cfg.walLevel; + pCreate->precision = pDb->cfg.precision; + pCreate->compression = pDb->cfg.compression; + pCreate->quorum = pDb->cfg.quorum; + pCreate->update = pDb->cfg.update; + pCreate->cacheLastRow = pDb->cfg.cacheLastRow; + pCreate->replica = pVgroup->replica; + pCreate->selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &pCreate->replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + free(pCreate); + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + pReplica->id = htonl(pVgidDnode->id); + pReplica->port = htons(pVgidDnode->port); + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + pCreate->selfIndex = v; + } + } + + if (pCreate->selfIndex == -1) { + free(pCreate); + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + return pCreate; +} + +SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { + SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg)); + if (pDrop == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pDrop->dnodeId = htonl(pDnode->id); + pDrop->vgId = htonl(pVgroup->vgId); + memcpy(pDrop->db, pDb->name, TSDB_FULL_DB_NAME_LEN); + pDrop->dbUid = htobe64(pDb->uid); + + return pDrop; +} + +static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) { + SSdb *pSdb = pMnode->pSdb; + int32_t allocedVnodes = 0; + void *pIter = NULL; + + while (allocedVnodes < pVgroup->replica) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + // todo + if (mndIsDnodeInReadyStatus(pMnode, pDnode)) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[allocedVnodes]; + pVgid->dnodeId = pDnode->id; + if (pVgroup->replica == 1) { + pVgid->role = TAOS_SYNC_STATE_LEADER; + } else { + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + } + allocedVnodes++; + } + sdbRelease(pSdb, pDnode); + } + + if (allocedVnodes != pVgroup->replica) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + return 0; +} + +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { + SVgObj *pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); + if (pVgroups == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t alloceVgroups = 0; + int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); + uint32_t hashMin = 0; + uint32_t hashMax = UINT32_MAX; + uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups; + + for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) { + SVgObj *pVgroup = &pVgroups[v]; + pVgroup->vgId = maxVgId++; + pVgroup->createdTime = taosGetTimestampMs(); + pVgroup->updateTime = pVgroups->createdTime; + pVgroup->version = 1; + pVgroup->dbUid = pDb->uid; + pVgroup->hashBegin = hashMin + hashInterval * v; + if (v == pDb->cfg.numOfVgroups - 1) { + pVgroup->hashEnd = hashMax; + } else { + pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1; + } + + memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN); + pVgroup->replica = pDb->cfg.replications; + + if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + free(pVgroups); + return -1; + } + + alloceVgroups++; + } + + *ppVgroups = pVgroups; + return 0; +} + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } + static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { SSdb *pSdb = pMnode->pSdb; @@ -203,30 +356,30 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tables"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; for (int32_t i = 0; i < pShow->replica; ++i) { - pShow->bytes[cols] = 4; + pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; } - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -273,10 +426,6 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols++; } - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pVgroup->compact; - cols++; - sdbRelease(pSdb, pVgroup); numOfRows++; } @@ -309,16 +458,16 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d96671d2bd..b4535597ee 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -32,6 +32,8 @@ int vnodeBufPoolRecycle(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); bool vnodeBufPoolIsFull(SVnode *pVnode); +SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index e6a88c7629..56e07aca10 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -19,7 +19,7 @@ #include "mallocator.h" #include "sync.h" #include "tcoding.h" -#include "tdlist.h" +#include "tlist.h" #include "tlockfree.h" #include "tmacro.h" #include "wal.h" diff --git a/source/common/inc/commonInt.h b/source/dnode/vnode/impl/inc/vnodeMAF.h similarity index 80% rename from source/common/inc/commonInt.h rename to source/dnode/vnode/impl/inc/vnodeMAF.h index 5b71f83faf..7aa405103c 100644 --- a/source/common/inc/commonInt.h +++ b/source/dnode/vnode/impl/inc/vnodeMAF.h @@ -13,19 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_INT_H_ -#define _TD_COMMON_INT_H_ +#ifndef _TD_VNODE_MAF_H_ +#define _TD_VNODE_MAF_H_ + +#include "vnode.h" #ifdef __cplusplus extern "C" { #endif -extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); - - +int vnodeOpenMAF(SVnode *pVnode); +void vnodeCloseMAF(SVnode *pVnode); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_INT_H_*/ +#endif /*_TD_VNODE_MAF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 99d4781df9..5999b08a7d 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -27,7 +27,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->capacity = capacity; pVMA->ssize = ssize; pVMA->lsize = lsize; - tSListInit(&(pVMA->nlist)); + TD_SLIST_INIT(&(pVMA->nlist)); pVMA->pNode = vArenaNodeNew(capacity); if (pVMA->pNode == NULL) { @@ -35,7 +35,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { return NULL; } - tSListPush(&(pVMA->nlist), pVMA->pNode); + TD_SLIST_PUSH(&(pVMA->nlist), pVMA->pNode); return pVMA; } @@ -44,7 +44,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - tSListPop(&(pVMA->nlist)); + TD_SLIST_POP(&(pVMA->nlist)); vArenaNodeFree(pNode); } @@ -55,7 +55,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { void vmaReset(SVMemAllocator *pVMA) { while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - tSListPop(&(pVMA->nlist)); + TD_SLIST_POP(&(pVMA->nlist)); vArenaNodeFree(pNode); } @@ -75,7 +75,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { return NULL; } - tSListPush(&(pVMA->nlist), pNode); + TD_SLIST_PUSH(&(pVMA->nlist), pNode); } ptr = pNode->ptr; diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 152a346f0a..228df6c0a4 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -24,10 +24,13 @@ struct SVBufPool { TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; - // MAF for submodules - // SMemAllocatorFactory maf; + // MAF for submodules to use + SMemAllocatorFactory *pMAF; }; +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA); + int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; @@ -36,8 +39,8 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tDListInit(&(pVnode->pBufPool->free)); - tDListInit(&(pVnode->pBufPool->incycle)); + TD_DLIST_INIT(&(pVnode->pBufPool->free)); + TD_DLIST_INIT(&(pVnode->pBufPool->incycle)); pVnode->pBufPool->inuse = NULL; @@ -51,9 +54,18 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tDListAppend(&(pVnode->pBufPool->free), pVMA); + TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } + pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); + if (pVnode->pBufPool->pMAF == NULL) { + // TODO: handle error + return -1; + } + pVnode->pBufPool->pMAF->impl = pVnode; + pVnode->pBufPool->pMAF->create = vBufPoolCreateMA; + pVnode->pBufPool->pMAF->destroy = vBufPoolDestroyMA; + return 0; } @@ -64,14 +76,14 @@ void vnodeCloseBufPool(SVnode *pVnode) { while (true) { SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); if (pVMA == NULL) break; - tDListPop(&(pVnode->pBufPool->incycle), pVMA); + TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); vmaDestroy(pVMA); } while (true) { SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); if (pVMA == NULL) break; - tDListPop(&(pVnode->pBufPool->free), pVMA); + TD_DLIST_POP(&(pVnode->pBufPool->free), pVMA); vmaDestroy(pVMA); } @@ -85,7 +97,7 @@ int vnodeBufPoolSwitch(SVnode *pVnode) { pVnode->pBufPool->inuse = NULL; - tDListAppend(&(pVnode->pBufPool->incycle), pvma); + TD_DLIST_APPEND(&(pVnode->pBufPool->incycle), pvma); return 0; } @@ -94,9 +106,9 @@ int vnodeBufPoolRecycle(SVnode *pVnode) { SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle)); ASSERT(pvma != NULL); - tDListPop(&(pBufPool->incycle), pvma); + TD_DLIST_POP(&(pBufPool->incycle), pvma); vmaReset(pvma); - tDListAppend(&(pBufPool->free), pvma); + TD_DLIST_APPEND(&(pBufPool->free), pvma); return 0; } @@ -109,7 +121,7 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { // TODO: add sem_wait and sem_post pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); if (pBufPool->inuse) { - tDListPop(&(pBufPool->free), pBufPool->inuse); + TD_DLIST_POP(&(pBufPool->free), pBufPool->inuse); break; } else { // tsem_wait(&(pBufPool->hasFree)); @@ -125,195 +137,50 @@ bool vnodeBufPoolIsFull(SVnode *pVnode) { return vmaIsFull(pVnode->pBufPool->inuse); } -#if 0 - -typedef enum { - // Heap allocator - E_V_HEAP_ALLOCATOR = 0, - // Arena allocator - E_V_ARENA_ALLOCATOR -} EVMemAllocatorT; - -typedef struct { - /* TODO */ -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - uint64_t size; - void * ptr; - char data[]; -} SVArenaNode; - -typedef struct { - uint64_t ssize; // step size - uint64_t lsize; // limit size - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - SVnode * pVnode; - SListNode *pNode; -} SVMAWrapper; - - -static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); -static void vBufPoolFreeNode(SListNode *pNode); -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); -static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); +SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; } /* ------------------------ STATIC METHODS ------------------------ */ -static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { - SListNode * pNode; - SVMemAllocator *pvma; - uint64_t msize; - uint64_t ssize = 4096; // TODO - uint64_t lsize = 1024; // TODO - - msize = sizeof(SListNode) + sizeof(SVMemAllocator); - if (type == E_V_ARENA_ALLOCATOR) { - msize += capacity; - } - - pNode = (SListNode *)calloc(1, msize); - if (pNode == NULL) { - // TODO: handle error - return NULL; - } - - pvma = (SVMemAllocator *)(pNode->data); - pvma->capacity = capacity; - pvma->type = type; - - switch (type) { - case E_V_ARENA_ALLOCATOR: - vArenaAllocatorInit(&(pvma->vaa), capacity, ssize, lsize); - break; - case E_V_HEAP_ALLOCATOR: - // vHeapAllocatorInit(&(pvma->vha)); - break; - default: - ASSERT(0); - } - - return pNode; -} - -static void vBufPoolFreeNode(SListNode *pNode) { - SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); - - switch (pvma->type) { - case E_V_ARENA_ALLOCATOR: - vArenaAllocatorClear(&(pvma->vaa)); - break; - case E_V_HEAP_ALLOCATOR: - // vHeapAllocatorClear(&(pvma->vha)); - break; - default: - break; - } - - free(pNode); -} - -static void *vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size) { - void *ptr = NULL; - - if (pvma->type == E_V_ARENA_ALLOCATOR) { - SVArenaAllocator *pvaa = &(pvma->vaa); - - if (POINTER_DISTANCE(pvaa->inuse->ptr, pvaa->inuse->data) + size > pvaa->inuse->size) { - SVArenaNode *pNode = (SVArenaNode *)malloc(sizeof(*pNode) + MAX(size, pvaa->ssize)); - if (pNode == NULL) { - // TODO: handle error - return NULL; - } - - pNode->prev = pvaa->inuse; - pNode->size = MAX(size, pvaa->ssize); - pNode->ptr = pNode->data; - - pvaa->inuse = pNode; - } - - ptr = pvaa->inuse->ptr; - pvaa->inuse->ptr = POINTER_SHIFT(ptr, size); - } else if (pvma->type == E_V_HEAP_ALLOCATOR) { - /* TODO */ - } - - return ptr; -} - -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf) { +typedef struct { SVnode * pVnode; - SMemAllocator * pma; - SVMemAllocator *pvma; - SVMAWrapper * pvmaw; + SVMemAllocator *pVMA; +} SVMAWrapper; - pVnode = (SVnode *)(pmaf->impl); - pma = (SMemAllocator *)calloc(1, sizeof(*pma) + sizeof(SVMAWrapper)); - if (pma == NULL) { - // TODO: handle error +static FORCE_INLINE void *vmaMaloocCb(SMemAllocator *pMA, uint64_t size) { + SVMAWrapper *pWrapper = (SVMAWrapper *)(pMA->impl); + + return vmaMalloc(pWrapper->pVMA, size); +} + +// TODO: Add atomic operations here +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { + SMemAllocator *pMA; + SVnode * pVnode = (SVnode *)(pMAF->impl); + SVMAWrapper * pWrapper; + + pMA = (SMemAllocator *)calloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); + if (pMA == NULL) { return NULL; } - pvmaw = (SVMAWrapper *)POINTER_SHIFT(pma, sizeof(*pma)); - // No allocator used currently - if (pVnode->pBufPool->inuse == NULL) { - while (listNEles(&(pVnode->pBufPool->free)) == 0) { - // TODO: wait until all released ro kill query - // tsem_wait(); - ASSERT(0); - } + pVnode->pBufPool->inuse->_ref.val++; + pWrapper = POINTER_SHIFT(pMA, sizeof(*pMA)); + pWrapper->pVnode = pVnode; + pWrapper->pVMA = pVnode->pBufPool->inuse; - pVnode->pBufPool->inuse = tdListPopHead(&(pVnode->pBufPool->free)); - pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - T_REF_INIT_VAL(pvma, 1); - } else { - pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - } + pMA->impl = pWrapper; + TD_MA_MALLOC_FUNC(pMA) = vmaMaloocCb; - T_REF_INC(pvma); - - pvmaw->pVnode = pVnode; - pvmaw->pNode = pVnode->pBufPool->inuse; - - pma->impl = pvmaw; - pma->malloc = NULL; - pma->calloc = NULL; /* TODO */ - pma->realloc = NULL; /* TODO */ - pma->free = NULL; /* TODO */ - pma->usage = NULL; /* TODO */ - - return pma; + return pMA; } -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { /* TODO */ - SVnode * pVnode = (SVnode *)(pmaf->impl); - SListNode * pNode = ((SVMAWrapper *)(pma->impl))->pNode; - SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { + SVMAWrapper * pWrapper = (SVMAWrapper *)(pMA->impl); + SVnode * pVnode = pWrapper->pVnode; + SVMemAllocator *pVMA = pWrapper->pVMA; - if (T_REF_DEC(pvma) == 0) { - if (pvma->type == E_V_ARENA_ALLOCATOR) { - SVArenaAllocator *pvaa = &(pvma->vaa); - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pNode = pvaa->inuse; - pvaa->inuse = pNode->prev; - /* code */ - } - - pvaa->inuse->ptr = pvaa->inuse->data; - } else if (pvma->type == E_V_HEAP_ALLOCATOR) { - } else { - ASSERT(0); - } - - // Move node from incycle to free - tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); - // tsem_post(); todo: sem_post + free(pMA); + if (--pVMA->_ref.val == 0) { + TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); + TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index a728de0ebb..7213e31cb4 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -25,6 +25,10 @@ int vnodeAsyncCommit(SVnode *pVnode) { pTask->execute = vnodeCommit; // TODO pTask->arg = pVnode; // TODO + tsdbPrepareCommit(pVnode->pTsdb); + // metaPrepareCommit(pVnode->pMeta); + // walPreapareCommit(pVnode->pWal); + vnodeScheduleTask(pTask); return 0; } @@ -32,6 +36,10 @@ int vnodeAsyncCommit(SVnode *pVnode) { int vnodeCommit(void *arg) { SVnode *pVnode = (SVnode *)arg; + metaCommit(pVnode->pMeta); + tqCommit(pVnode->pTq); + tsdbCommit(pVnode->pTq); + vnodeBufPoolRecycle(pVnode); // TODO return 0; diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 59e3bae5d7..70d9c7d4b0 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -94,7 +94,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open meta sprintf(dir, "%s/meta", pVnode->path); - pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg)); + pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pMeta == NULL) { // TODO: handle error return -1; @@ -102,7 +102,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, &(pVnode->config.tsdbCfg)); + pVnode->pTsdb = tsdbOpen(dir, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; @@ -110,7 +110,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // TODO: Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, NULL); + pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; @@ -131,7 +131,9 @@ static int vnodeOpenImpl(SVnode *pVnode) { static void vnodeCloseImpl(SVnode *pVnode) { if (pVnode) { vnodeCloseBufPool(pVnode); - tsdbClose(pVnode->pTsdb); metaClose(pVnode->pMeta); + tsdbClose(pVnode->pTsdb); + tqClose(pVnode->pTq); + walClose(pVnode->pWal); } } \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 964cbe77da..fae96ae22c 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -34,7 +34,7 @@ int vnodeInit(uint16_t nthreads) { pthread_mutex_init(&(vnodeMgr.mutex), NULL); pthread_cond_init(&(vnodeMgr.hasTask), NULL); - tDListInit(&(vnodeMgr.queue)); + TD_DLIST_INIT(&(vnodeMgr.queue)); for (uint16_t i = 0; i < nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); @@ -77,7 +77,7 @@ void vnodeClear() { int vnodeScheduleTask(SVnodeTask* pTask) { pthread_mutex_lock(&(vnodeMgr.mutex)); - tDListAppend(&(vnodeMgr.queue), pTask); + TD_DLIST_APPEND(&(vnodeMgr.queue), pTask); pthread_cond_signal(&(vnodeMgr.hasTask)); @@ -101,7 +101,7 @@ static void* loop(void* arg) { pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); } } else { - tDListPop(&(vnodeMgr.queue), pTask); + TD_DLIST_POP(&(vnodeMgr.queue), pTask); break; } } diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/impl/src/vnodeRequest.c index be5f5c890c..249bde4e56 100644 --- a/source/dnode/vnode/impl/src/vnodeRequest.c +++ b/source/dnode/vnode/impl/src/vnodeRequest.c @@ -25,9 +25,10 @@ int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type) { switch (type) { case TSDB_MSG_TYPE_CREATE_TABLE: tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); + break; + case TSDB_MSG_TYPE_SUBMIT: /* code */ break; - default: break; } diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp new file mode 100644 index 0000000000..e218886231 --- /dev/null +++ b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp @@ -0,0 +1,2 @@ +// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest +// https://github.com/google/benchmark \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index f0bca3aa2e..6f2e6f721a 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -14,7 +14,7 @@ #include "vnode.h" -static STSchema *createBasicSchema() { +static STSchema *vtCreateBasicSchema() { STSchemaBuilder sb; STSchema * pSchema = NULL; @@ -32,7 +32,7 @@ static STSchema *createBasicSchema() { return pSchema; } -static STSchema *createBasicTagSchema() { +static STSchema *vtCreateBasicTagSchema() { STSchemaBuilder sb; STSchema * pSchema = NULL; @@ -50,7 +50,7 @@ static STSchema *createBasicTagSchema() { return pSchema; } -static SKVRow createBasicTag() { +static SKVRow vtCreateBasicTag() { SKVRowBuilder rb; SKVRow pTag; @@ -71,118 +71,203 @@ static SKVRow createBasicTag() { return pTag; } -#if 0 -TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { - tb_uid_t suid = 1638166374163; - STSchema *pSchema = createBasicSchema(); - STSchema *pTagSchema = createBasicTagSchema(); - char tbname[128] = "st"; - char * buffer = new char[1024]; - void * pBuf = (void *)buffer; +static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { + SRpcMsg * pMsg; + STSchema *pSchema; + STSchema *pTagSchema; + int zs; + void * pBuf; + + pSchema = vtCreateBasicSchema(); + pTagSchema = vtCreateBasicTagSchema(); + SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); + zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); + pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->contLen = zs; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); + + pBuf = pMsg->pCont; vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + META_CLEAR_TB_CFG(&vCreateSTbReq); - SVnodeReq decoded_req; + tdFreeSchema(pSchema); + tdFreeSchema(pTagSchema); - vnodeParseReq(buffer, &decoded_req, TSDB_MSG_TYPE_CREATE_TABLE); - - int k = 10; + *ppMsg = pMsg; } -#endif -TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { +static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { + SRpcMsg *pMsg; + int tz; + SKVRow pTag = vtCreateBasicTag(); + + SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); + + tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); + pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->contLen = tz; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); + void *pBuf = pMsg->pCont; + + vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + META_CLEAR_TB_CFG(&vCreateCTbReq); + free(pTag); + + *ppMsg = pMsg; +} + +static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { + // TODO +} + +static void vtBuildSubmitReq(SRpcMsg **ppMsg) { + SRpcMsg * pMsg; + SSubmitMsg *pSubmitMsg; + SSubmitBlk *pSubmitBlk; + int tz = 1024; // TODO + + pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); + pMsg->msgType = TSDB_MSG_TYPE_SUBMIT; + pMsg->contLen = tz; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); + + // For submit msg header + pSubmitMsg = (SSubmitMsg *)(pMsg->pCont); + // pSubmitMsg->header.contLen = 0; + // pSubmitMsg->header.vgId = 0; + // pSubmitMsg->length = 0; + pSubmitMsg->numOfBlocks = 1; + + // For submit blk + pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks); + pSubmitBlk->uid = 0; + pSubmitBlk->tid = 0; + pSubmitBlk->padding = 0; + pSubmitBlk->sversion = 0; + pSubmitBlk->dataLen = 0; + pSubmitBlk->numOfRows = 0; + + // For row batch + + *ppMsg = pMsg; +} + +static void vtClearMsgBatch(SArray *pMsgArr) { + SRpcMsg *pMsg; + for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { + pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i); + free(pMsg); + } + + taosArrayClear(pMsgArr); +} + +TEST(vnodeApiTest, vnode_simple_create_table_test) { + tb_uid_t suid = 1638166374163; + SRpcMsg *pMsg; + SArray * pMsgArr = NULL; + SVnode * pVnode; + int rcode; + int ntables = 1000000; + int batch = 10; + char tbname[128]; + + pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg)); + vnodeDestroy("vnode1"); + GTEST_ASSERT_GE(vnodeInit(2), 0); + + // CREATE AND OPEN A VNODE + pVnode = vnodeOpen("vnode1", NULL); + ASSERT_NE(pVnode, nullptr); + + // CREATE A SUPER TABLE + sprintf(tbname, "st"); + vtBuildCreateStbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); + + // CREATE A LOT OF CHILD TABLES + for (int i = 0; i < ntables / batch; i++) { + // Build request batch + for (int j = 0; j < batch; j++) { + sprintf(tbname, "ct%d", i * batch + j + 1); + vtBuildCreateCtbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + } + + // Process request batch + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + ASSERT_EQ(rcode, 0); + + // Clear request batch + vtClearMsgBatch(pMsgArr); + } + + // CLOSE THE VNODE + vnodeClose(pVnode); + vnodeClear(); + + taosArrayDestroy(pMsgArr); +} + +TEST(vnodeApiTest, vnode_simple_insert_test) { + const char *vname = "vnode2"; + char tbname[128]; + tb_uid_t suid = 1638166374163; + SRpcMsg * pMsg; + SArray * pMsgArr; + int rcode; + SVnode * pVnode; + int batch = 1; + int loop = 1000000; + + pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); + + vnodeDestroy(vname); GTEST_ASSERT_GE(vnodeInit(2), 0); - // Create and open a vnode - SVnode *pVnode = vnodeOpen("vnode1", NULL); - ASSERT_NE(pVnode, nullptr); + // Open a vnode + pVnode = vnodeOpen(vname, NULL); + GTEST_ASSERT_NE(pVnode, nullptr); - tb_uid_t suid = 1638166374163; - { - // Create a super table - STSchema *pSchema = createBasicSchema(); - STSchema *pTagSchema = createBasicTagSchema(); - char tbname[128] = "st"; + // 1. CREATE A SUPER TABLE + sprintf(tbname, "st"); + vtBuildCreateStbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); - SArray * pMsgs = (SArray *)taosArrayInit(1, sizeof(SRpcMsg *)); - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); + // 2. CREATE A CHILD TABLE + sprintf(tbname, "t0"); + vtBuildCreateCtbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); - int zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - pMsg->contLen = zs; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); - - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateSTbReq); - - taosArrayPush(pMsgs, &(pMsg)); - - vnodeProcessWMsgs(pVnode, pMsgs); - - free(pMsg); - taosArrayDestroy(pMsgs); - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - } - - { - // Create some child tables - int ntables = 1000000; - int batch = 10; - for (int i = 0; i < ntables / batch; i++) { - SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); - for (int j = 0; j < batch; j++) { - SKVRow pTag = createBasicTag(); - char tbname[128]; - sprintf(tbname, "tb%d", i * batch + j); - SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - - int tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateCTbReq); - free(pTag); - - taosArrayPush(pMsgs, &(pMsg)); - } - - vnodeProcessWMsgs(pVnode, pMsgs); - - for (int j = 0; j < batch; j++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayPop(pMsgs); - free(pMsg); - } - - taosArrayDestroy(pMsgs); - - // std::cout << "the " << i << "th batch is created" << std::endl; + // 3. WRITE A LOT OF TIME-SERIES DATA + for (int j = 0; j < loop; j++) { + for (int i = 0; i < batch; i++) { + vtBuildSubmitReq(&pMsg); + taosArrayPush(pMsgArr, &pMsg); } + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); } // Close the vnode vnodeClose(pVnode); - vnodeClear(); -} -TEST(vnodeApiTest, DISABLED_vnode_process_create_table) { - STSchema * pSchema = NULL; - STSchema * pTagSchema = NULL; - char stname[15]; - SVCreateTableReq pReq = META_INIT_STB_CFG(stname, UINT32_MAX, UINT32_MAX, 0, pSchema, pTagSchema); - - int k = 10; - - META_CLEAR_TB_CFG(pReq); -} + taosArrayDestroy(pMsgArr); +} \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index 0204031e76..e1c15af5aa 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -34,7 +34,7 @@ extern "C" { struct SMeta { char* path; SMetaCfg options; - SMetaDB* pDB; + SMetaDB* pDB; SMetaIdx* pIdx; SMetaCache* pCache; STbUidGenerator uidGnrt; diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index 53055dcea3..b6aa029b19 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -17,27 +17,27 @@ #include "metaDef.h" -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions); +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); static void metaFree(SMeta *pMeta); static int metaOpenImpl(SMeta *pMeta); static void metaCloseImpl(SMeta *pMeta); -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaOptions) { +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) { SMeta *pMeta = NULL; // Set default options - if (pMetaOptions == NULL) { - pMetaOptions = &defaultMetaOptions; + if (pMetaCfg == NULL) { + pMetaCfg = &defaultMetaOptions; } // Validate the options - if (metaValidateOptions(pMetaOptions) < 0) { + if (metaValidateOptions(pMetaCfg) < 0) { // TODO: deal with error return NULL; } // Allocate handle - pMeta = metaNew(path, pMetaOptions); + pMeta = metaNew(path, pMetaCfg, pMAF); if (pMeta == NULL) { // TODO: handle error return NULL; @@ -65,7 +65,7 @@ void metaClose(SMeta *pMeta) { void metaRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions) { +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) { SMeta *pMeta; size_t psize = strlen(path); @@ -80,7 +80,8 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions) { return NULL; } - metaOptionsCopy(&(pMeta->options), pMetaOptions); + metaOptionsCopy(&(pMeta->options), pMetaCfg); + pMeta->pmaf = pMAF; return pMeta; }; diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index 7c593cb4c7..b1375c9477 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -18,7 +18,7 @@ #include "mallocator.h" #include "taosmsg.h" -#include "tdlist.h" +#include "tlist.h" #include "thash.h" #include "tskiplist.h" diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index b124197736..a747c7333e 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -15,7 +15,19 @@ #include "tsdbDef.h" +int tsdbPrepareCommit(STsdb *pTsdb) { + if (pTsdb->mem == NULL) return 0; + + // tsem_wait(&(pTsdb->canCommit)); + ASSERT(pTsdb->imem == NULL); + + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; +} + int tsdbCommit(STsdb *pTsdb) { // TODO + pTsdb->imem = NULL; + // tsem_post(&(pTsdb->canCommit)); return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index 2fe7a61930..d67d45660d 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -15,27 +15,27 @@ #include "tsdbDef.h" -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions); +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); -STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbOptions) { +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; // Set default TSDB Options - if (pTsdbOptions == NULL) { - pTsdbOptions = &defautlTsdbOptions; + if (pTsdbCfg == NULL) { + pTsdbCfg = &defautlTsdbOptions; } // Validate the options - if (tsdbValidateOptions(pTsdbOptions) < 0) { + if (tsdbValidateOptions(pTsdbCfg) < 0) { // TODO: handle error return NULL; } // Create the handle - pTsdb = tsdbNew(path, pTsdbOptions); + pTsdb = tsdbNew(path, pTsdbCfg, pMAF); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -62,7 +62,7 @@ void tsdbClose(STsdb *pTsdb) { void tsdbRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions) { +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); @@ -72,7 +72,8 @@ static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions) { } pTsdb->path = strdup(path); - tsdbOptionsCopy(&(pTsdb->options), pTsdbOptions); + tsdbOptionsCopy(&(pTsdb->options), pTsdbCfg); + pTsdb->pmaf = pMAF; return pTsdb; } diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index e3d1f8673e..7b0df18f5a 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -50,7 +50,7 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { pMA = (*pMAF->create)(pMAF); ASSERT(pMA != NULL); - pMemTable = (STsdbMemTable *)((*pMA->malloc)(pMA, sizeof(*pMemTable))); + pMemTable = (STsdbMemTable *)TD_MA_MALLOC(pMA, sizeof(*pMemTable)); if (pMemTable == NULL) { (*pMAF->destroy)(pMAF, pMA); return NULL; @@ -62,7 +62,7 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { pMemTable->keyMax = TSKEY_MIN; pMemTable->nRow = 0; pMemTable->pMA = pMA; - tSListInit(&(pMemTable->list)); + TD_SLIST_INIT(&(pMemTable->list)); // TODO return pMemTable; @@ -71,7 +71,7 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { SMemAllocator *pMA = pMemTable->pMA; - if (pMA->free) { + if (TD_MA_FREE_FUNC(pMA) != NULL) { // TODO ASSERT(0); } @@ -81,12 +81,12 @@ void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { SMemAllocator *pMA = pMemTable->pMA; - STbData * pTbData = (STbData *)((*pMA->malloc)(pMA, sizeof(*pTbData))); + STbData * pTbData = (STbData *)TD_MA_MALLOC(pMA, sizeof(*pTbData)); if (pTbData == NULL) { // TODO } - tSListPush(&(pMemTable->list), pTbData); + TD_SLIST_PUSH(&(pMemTable->list), pTbData); return 0; } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 57a5023807..636cf0a9a8 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -10,4 +10,4 @@ add_subdirectory(catalog) add_subdirectory(executor) add_subdirectory(planner) add_subdirectory(function) -add_subdirectory(query) +add_subdirectory(qcom) diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index e6311152d6..f47e105b8a 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( catalog - PRIVATE os util common transport query + PRIVATE os util transport qcom ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 455c82b1bc..7b26079f11 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -70,9 +70,10 @@ extern int32_t ctgDebugFlag; #define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) -#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) -#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0) -#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) +#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0) +#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 9fdac36060..1134c7763a 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -49,26 +49,35 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen); - if (code) { - return code; + CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen)); + + char *pMsg = rpcMallocCont(msgLen); + if (NULL == pMsg) { + ctgError("rpc malloc %d failed", msgLen); + tfree(msg); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + memcpy(pMsg, msg, msgLen); + + tfree(msg); + SRpcMsg rpcMsg = { .msgType = TSDB_MSG_TYPE_USE_DB, - .pCont = msg, + .pCont = pMsg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - - code = queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - return code; + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for use db, code:%x", rpcRsp.code); + CTG_ERR_RET(rpcRsp.code); } + CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen)); + return TSDB_CODE_SUCCESS; } @@ -99,14 +108,14 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, if ((*stbMeta)->suid != tbMeta->suid) { ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); *pTableMeta = calloc(1, metaSize); if (NULL == *pTableMeta) { ctgError("calloc size[%d] failed", metaSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta)); @@ -116,7 +125,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, *pTableMeta = calloc(1, metaSize); if (NULL == *pTableMeta) { ctgError("calloc size[%d] failed", metaSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(*pTableMeta, tbMeta, metaSize); @@ -140,7 +149,7 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } char tbFullName[TSDB_TABLE_FNAME_LEN]; @@ -152,10 +161,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); - if (code) { - return code; - } + CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen)); SRpcMsg rpcMsg = { .msgType = TSDB_MSG_TYPE_TABLE_META, @@ -169,16 +175,13 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE ctgGenEpSet(&epSet, vgroupInfo); rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); - + if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code); - return rpcRsp.code; + ctgError("error rsp for table meta, code:%x", rpcRsp.code); + CTG_ERR_RET(rpcRsp.code); } - code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - return code; - } + CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen)); return TSDB_CODE_SUCCESS; } @@ -204,7 +207,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * if (NULL == taosArrayPush(vgroupList, vgInfo)) { ctgError("taosArrayPush failed"); - break; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } pIter = taosHashIterate(dbInfo->vgInfo, pIter); @@ -218,7 +221,7 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, co int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); if (vgNum <= 0) { ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); - return TSDB_CODE_TSC_DB_NOT_SELECTED; + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); } tableNameHashFp fp = NULL; @@ -245,7 +248,7 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, co if (NULL == vgInfo) { ctgError("no hash range found for hashvalue[%u]", hashValue); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } *pVgroup = *vgInfo; @@ -253,53 +256,9 @@ int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, co return TSDB_CODE_SUCCESS; } - -int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo dbInfo = {0}; - int32_t code = 0; - int32_t vgId = 0; - - CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); - - if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) { - ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo); - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup)); - - return code; -} - - - -STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) { - assert(pChild != NULL); - int32_t total = pChild->numOfColumns + pChild->numOfTags; - - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); - pTableMeta->tableType = TSDB_SUPER_TABLE; - pTableMeta->tableInfo.numOfTags = pChild->numOfTags; - pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; - pTableMeta->tableInfo.precision = pChild->precision; - - pTableMeta->uid = pChild->suid; - pTableMeta->tversion = pChild->tversion; - pTableMeta->sversion = pChild->sversion; - - memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); - - int32_t num = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < num; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; @@ -318,7 +277,7 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* if (0 == exist) { ctgError("get table meta from cache failed, but fetch succeed"); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } return TSDB_CODE_SUCCESS; @@ -328,19 +287,19 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { if (output->metaNum != 1 && output->metaNum != 2) { ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == output->tbMeta) { ctgError("no valid table meta got from meta rsp"); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == pCatalog->tableCache.cache) { pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -348,13 +307,13 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.stableCache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -392,7 +351,7 @@ error_exit: pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t catalogInit(SCatalogCfg *cfg) { @@ -414,12 +373,12 @@ int32_t catalogInit(SCatalogCfg *cfg) { int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { if (NULL == clusterId || NULL == catalogHandle) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == ctgMgmt.pCluster) { ctgError("cluster cache are not ready"); - return TSDB_CODE_CTG_NOT_READY; + CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } size_t clen = strlen(clusterId); @@ -433,7 +392,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) clusterCtg = calloc(1, sizeof(*clusterCtg)); if (NULL == clusterCtg) { ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg)); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; @@ -441,7 +400,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { ctgError("put cluster %s cache to hash failed", clusterId); tfree(clusterCtg); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } *catalogHandle = clusterCtg; @@ -451,7 +410,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { if (NULL == pCatalog || NULL == dbName || NULL == version) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == pCatalog->dbCache.cache) { @@ -472,7 +431,7 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (dbInfo->vgVersion < 0) { @@ -488,7 +447,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->dbCache.cache) { ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } else { SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); @@ -500,7 +459,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) { ctgError("push to vgroup hash cache failed"); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } return TSDB_CODE_SUCCESS; @@ -511,11 +470,10 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; - int32_t code = 0; if (0 == forceUpdate) { CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist)); @@ -540,7 +498,7 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* *dbInfo = DbOut.dbVgroup; } - return code; + return TSDB_CODE_SUCCESS; } int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { @@ -549,12 +507,12 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) { if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; - CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); + CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); STableMetaOutput output = {0}; @@ -571,9 +529,9 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); } -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { +int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } STableMeta *tbMeta = NULL; @@ -591,7 +549,7 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe int32_t vgId = tbMeta->vgId; if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("vgId[%d] not found in vgroup list", vgId); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) { @@ -603,13 +561,31 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe _return: tfree(tbMeta); - return code; + CTG_RET(code); +} + + +int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + SDBVgroupInfo dbInfo = {0}; + int32_t code = 0; + int32_t vgId = 0; + + CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); + + if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) { + ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup)); + + CTG_RET(code); } int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; @@ -621,7 +597,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); if (NULL == pRsp->pTableMeta) { ctgError("taosArrayInit num[%d] failed", tbNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -655,9 +631,19 @@ _return: taosArrayDestroy(pRsp->pTableMeta); } - return code; + CTG_RET(code); } +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeEpSet) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + + return TSDB_CODE_SUCCESS; +} + + void catalogDestroy(void) { if (ctgMgmt.pCluster) { taosHashCleanup(ctgMgmt.pCluster); //TBD diff --git a/source/libs/catalog/test/CMakeLists.txt b/source/libs/catalog/test/CMakeLists.txt index 527156f176..3c7418bdcc 100644 --- a/source/libs/catalog/test/CMakeLists.txt +++ b/source/libs/catalog/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(catalogTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( catalogTest - PUBLIC os util common catalog transport gtest query + PUBLIC os util common catalog transport gtest qcom taos ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index f495451091..e14c58d412 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include +#include #include #include #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -23,130 +23,103 @@ #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" -#include "taos.h" +#include "taos.h" #include "tdef.h" -#include "tvariant.h" -#include "catalog.h" - +#include "tvariant.h" +#include "catalog.h" +#include "tep.h" +#include "trpc.h" + +typedef struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; +} SAppInstInfo; + +typedef struct STscObj { + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + uint32_t connId; + uint64_t id; // ref ID returned by taosAddRef +// struct SSqlObj *sqlList; + void *pTransporter; + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj from this tscObj + SAppInstInfo *pAppInfo; +} STscObj; + namespace { - - + +void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.db1"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp); + + ASSERT_EQ(rpcRsp.code, 0); } - -TEST(testCase, normalCase) { - char *clusterId = "cluster1"; - struct SCatalog* pCtg = NULL; - - int32_t code = catalogInit(NULL); - ASSERT_EQ(code, 0); - - code = catalogGetHandle(clusterId, &pCtg); - ASSERT_EQ(code, 0); - - -} - -/* -TEST(testCase, normalCase) { - SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; +} - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +TEST(testCase, normalCase) { + STscObj* pConn = (STscObj *)taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + + msgInit(); + + sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - SCatalogReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + code = catalogGetHandle(clusterId, &pCtg); + ASSERT_EQ(code, 0); - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); + code = catalogGetTableHashVgroup(pCtg, pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet, dbname, tablename, &vgInfo); + ASSERT_EQ(code, 0); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_EQ(ret, 0); - - SArray* pExprList = pQueryInfo->exprList[0]; - - int32_t num = tsCompatibleModel? 2:1; - ASSERT_EQ(taosArrayGetSize(pExprList), num); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p1->base.pColumns->uid, 110); - ASSERT_EQ(p1->base.numOfParams, 1); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); - ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.interBytes, 16); - - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); - ASSERT_STREQ(p1->pExpr->_function.functionName, "top"); - - tExprNode* pParam = p1->pExpr->_function.pChild[0]; - - ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); - - struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); - - char* str = NULL; - queryPlanToString(n, &str); - printf("%s\n", str); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); + taos_close(pConn); } -TEST(testCase, displayPlan) { - generateLogicplan("select count(*) from `t.1abc`"); - generateLogicplan("select count(*)+ 22 from `t.1abc`"); - generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30"); - generateLogicplan("select count(*) from `t.1abc` group by a"); - generateLogicplan("select count(A+B) from `t.1abc` group by a"); - generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); - generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); - generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s) order by 1 desc "); - generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); - generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); - generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); - generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20"); - generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)"); - generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)"); - // order by + group by column + limit offset - generateLogicplan("select top(a, 20) k from `t.1abc` order by k asc limit 3 offset 1"); - - // fill - generateLogicplan("select min(a) from `t.1abc` where ts>now and ts @@ -28,16 +31,42 @@ extern "C" { #endif +typedef enum {kTypeValue, kTypeDeletion} STermValueType ; + +typedef struct SIndexStat { + int32_t totalAdded; // + int32_t totalDeled; // + int32_t totalUpdated; // + int32_t totalTerms; // + int32_t distinctCol; // distinct column +} SIndexStat; + struct SIndex { #ifdef USE_LUCENE index_t *index; #endif + void *cache; + void *tindex; + SHashObj *colObj;// < field name, field id> + + int64_t suid; // current super table id, -1 is normal table + int colId; // field id allocated to cache + int32_t cVersion; // current version allocated to cache + + SIndexStat stat; + pthread_mutex_t mtx; }; struct SIndexOpts { #ifdef USE_LUCENE void *opts; -#endif +#endif + +#ifdef USE_INVERTED_INDEX + int32_t cacheSize; // MB + // add cache module later +#endif + }; struct SIndexMultiTermQuery { @@ -47,21 +76,21 @@ struct SIndexMultiTermQuery { // field and key; typedef struct SIndexTerm { - char *key; - int32_t nKey; - char *val; - int32_t nVal; + int64_t suid; + SIndexOperOnColumn operType; // oper type, add/del/update + uint8_t colType; // term data type, str/interger/json + char *colName; + int32_t nColName; + char *colVal; + int32_t nColVal; } SIndexTerm; typedef struct SIndexTermQuery { - SIndexTerm* field_value; - EIndexQueryType type; + SIndexTerm* term; + EIndexQueryType qType; } SIndexTermQuery; -SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal); -void indexTermDestroy(SIndexTerm *p); - #define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) #define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h new file mode 100644 index 0000000000..97a7b835f6 --- /dev/null +++ b/source/libs/index/inc/index_cache.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_CACHE_H__ +#define __INDEX_CACHE_H__ + +#include "index.h" +#include "indexInt.h" +#include "tlockfree.h" +#include "tskiplist.h" +// ----------------- key structure in skiplist --------------------- + +/* A data row, the format is like below: + * content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->| + * len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| + */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct IndexCache { + T_REF_DECLARE() + SSkipList *skiplist; +} IndexCache; + + +// +IndexCache *indexCacheCreate(); + +void indexCacheDestroy(void *cache); + +int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); + +//int indexCacheGet(void *cache, uint64_t *rst); +int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); + +#ifdef __cplusplus +} +#endif + + + +#endif diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 20037f829a..0dcc25831c 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -315,7 +315,6 @@ typedef struct StreamWithStateResult { FstSlice data; FstOutput out; void *state; - } StreamWithStateResult; StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h new file mode 100644 index 0000000000..c3f4bd25e5 --- /dev/null +++ b/source/libs/index/inc/index_tfile.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_TFILE_H__ +#define __INDEX_TFILE_H__ + +#include "index.h" +#include "indexInt.h" +#include "tlockfree.h" +#include "tskiplist.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct IndexTFile { + T_REF_DECLARE() +} IndexTFile; + + + +IndexTFile *indexTFileCreate(); + + +int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); + +#ifdef __cplusplus +} + + +#endif + + + +#endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f8f4311a4a..ec83e84a3b 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -15,47 +15,81 @@ #include "index.h" #include "indexInt.h" +#include "index_cache.h" +#include "index_tfile.h" +#include "tdef.h" #ifdef USE_LUCENE #include "lucene++/Lucene_c.h" #endif -static pthread_once_t isInit = PTHREAD_ONCE_INIT; +static int uidCompare(const void *a, const void *b) { + uint64_t u1 = *(uint64_t *)a; + uint64_t u2 = *(uint64_t *)b; + if (u1 == u2) { return 0; } + else { return u1 < u2 ? -1 : 1; } +} +typedef struct SIdxColInfo { + int colId; // generated by index internal + int cVersion; +} SIdxColInfo; +static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); -SIndex *indexOpen(SIndexOpts *opts, const char *path) { + +static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); +static int indexMergeCacheIntoTindex(SIndex *sIdx); + +static void indexInterResultsDestroy(SArray *results); +static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); + +int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { pthread_once(&isInit, indexInit); + SIndex *sIdx = calloc(1, sizeof(SIndex)); + if (sIdx == NULL) { return -1; } + #ifdef USE_LUCENE index_t *index = index_open(path); - SIndex *p = malloc(sizeof(SIndex)); - p->index = index; - return p; + sIdx->index = index; #endif - return NULL; + + sIdx->cache = (void*)indexCacheCreate(); + sIdx->tindex = NULL; + sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + sIdx->colId = 1; + sIdx->cVersion = 1; + pthread_mutex_init(&sIdx->mtx, NULL); + + *index = sIdx; + return 0; } -void indexClose(SIndex *index) { +void indexClose(SIndex *sIdx) { #ifdef USE_LUCENE - index_close(index->index); - index->index = NULL; + index_close(sIdex->index); + sIdx->index = NULL; #endif - free(index); - return; +#ifdef USE_INVERTED_INDEX + indexCacheDestroy(sIdx->cache); + taosHashCleanup(sIdx->colObj); + pthread_mutex_destroy(&sIdx->mtx); +#endif + free(sIdx); + return; } -#ifdef USE_LUCENE -#endif -int indexPut(SIndex *index, SArray* field_vals, int uid) { +int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) { + #ifdef USE_LUCENE index_document_t *doc = index_document_create(); char buf[16] = {0}; sprintf(buf, "%d", uid); - for (int i = 0; i < taosArrayGetSize(field_vals); i++) { - SIndexTerm *p = taosArrayGetP(field_vals, i); + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); } index_document_add(doc, NULL, 0, buf, strlen(buf), 0); @@ -63,8 +97,39 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) { index_put(index->index, doc); index_document_destroy(doc); #endif - return 1; +#ifdef USE_INVERTED_INDEX + + //TODO(yihao): reduce the lock range + pthread_mutex_lock(&index->mtx); + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); + SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + if (fi == NULL) { + SIdxColInfo tfi = {.colId = index->colId}; + index->cVersion++; + index->colId++; + taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); + } else { + //TODO, del + } + } + pthread_mutex_unlock(&index->mtx); + + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); + SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + assert(fi != NULL); + int32_t colId = fi->colId; + int32_t version = index->cVersion; + int ret = indexCachePut(index->cache, p, colId, version, uid); + if (ret != 0) { + return ret; + } + } +#endif + + return 0; } int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { #ifdef USE_LUCENE @@ -101,14 +166,39 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result free(fields); free(keys); free(types); +#endif + +#ifdef USE_INVERTED_INDEX + EIndexOperatorType opera = multiQuerys->opera; // relation of querys + + SArray *interResults = taosArrayInit(4, POINTER_BYTES); + int nQuery = taosArrayGetSize(multiQuerys->query); + for (size_t i = 0; i < nQuery; i++) { + SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); + SArray *tResult = NULL; + indexTermSearch(index, qTerm, &tResult); + taosArrayPush(interResults, (void *)&tResult); + } + indexMergeFinalResults(interResults, opera, result); + indexInterResultsDestroy(interResults); + #endif return 1; } + + int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { +#ifdef USE_INVERTED_INDEX +#endif + return 1; } -int indexRebuild(SIndex *index, SIndexOpts *opts); +int indexRebuild(SIndex *index, SIndexOpts *opts) { +#ifdef USE_INVERTED_INDEX +#endif + +} SIndexOpts *indexOptsCreate() { @@ -129,61 +219,135 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } p->opera = opera; - p->query = taosArrayInit(1, sizeof(SIndexTermQuery)); + p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); - indexTermDestroy(p->field_value); + indexTermDestroy(p->term); } taosArrayDestroy(pQuery->query); free(pQuery); }; -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type){ - SIndexTerm *t = indexTermCreate(field, nFields, value, nValue); - if (t == NULL) {return -1;} - SIndexTermQuery q = {.type = type, .field_value = t}; +int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){ + SIndexTermQuery q = {.qType = qType, .term = term}; taosArrayPush(pQuery->query, &q); return 0; } -SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal) { - SIndexTerm *t = (SIndexTerm *)malloc(sizeof(SIndexTerm)); - t->key = (char *)calloc(nKey + 1, 1); - memcpy(t->key, key, nKey); - t->nKey = nKey; +SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) { + SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); + if (t == NULL) { return NULL; } - t->val = (char *)calloc(nVal + 1, 1); - memcpy(t->val, val, nVal); - t->nVal = nVal; + t->suid = suid; + t->operType= oper; + t->colType = colType; + + t->colName = (char *)calloc(1, nColName + 1); + memcpy(t->colName, colName, nColName); + t->nColName = nColName; + + t->colVal = (char *)calloc(1, nColVal + 1); + memcpy(t->colVal, colVal, nColVal); + t->nColVal = nColVal; return t; } void indexTermDestroy(SIndexTerm *p) { - free(p->key); - free(p->val); + free(p->colName); + free(p->colVal); free(p); } -SArray *indexMultiTermCreate() { +SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } -int indexMultiTermAdd(SArray *array, const char *field, int32_t nField, const char *val, int32_t nVal) { - SIndexTerm *term = indexTermCreate(field, nField, val, nVal); - if (term == NULL) { return -1; } - taosArrayPush(array, &term); +int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { + taosArrayPush(terms, &term); return 0; } -void indexMultiTermDestroy(SArray *array) { - for (int32_t i = 0; i < taosArrayGetSize(array); i++) { - SIndexTerm *p = taosArrayGetP(array, i); +void indexMultiTermDestroy(SIndexMultiTerm *terms) { + for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { + SIndexTerm *p = taosArrayGetP(terms, i); indexTermDestroy(p); } - taosArrayDestroy(array); + taosArrayDestroy(terms); } void indexInit() { //do nothing } +static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { + int32_t version = -1; + int16_t colId = -1; + SIdxColInfo *colInfo = NULL; + + SIndexTerm *term = query->term; + const char *colName = term->colName; + int32_t nColName = term->nColName; + + pthread_mutex_lock(&sIdx->mtx); + colInfo = taosHashGet(sIdx->colObj, colName, nColName); + if (colInfo == NULL) { + pthread_mutex_unlock(&sIdx->mtx); + return -1; + } + colId = colInfo->colId; + version = colInfo->cVersion; + pthread_mutex_unlock(&sIdx->mtx); + + *result = taosArrayInit(4, sizeof(uint64_t)); + //TODO: iterator mem and tidex + STermValueType s; + if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) { + if (s == kTypeDeletion) { + indexInfo("col: %s already drop by other opera", term->colName); + // coloum already drop by other oper, no need to query tindex + return 0; + } else { + if (0 != indexTFileSearch(sIdx->tindex, query, *result)) { + indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); + return -1; + } + } + } else { + indexError("corrupt at index(cache) col:%s val: %s", term->colName, term->colVal); + return -1; + } + return 0; +} +static void indexInterResultsDestroy(SArray *results) { + if (results == NULL) { return; } + + size_t sz = taosArrayGetSize(results); + for (size_t i = 0; i < sz; i++) { + SArray *p = taosArrayGetP(results, i); + taosArrayDestroy(p); + } + taosArrayDestroy(results); + +} +static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { + //refactor, merge interResults into fResults by oType + SArray *first = taosArrayGetP(interResults, 0); + taosArraySort(first, uidCompare); + if (oType == MUST) { + + } else if (oType == SHOULD) { + + // tag1 condistion || tag2 condition + } else if (oType == NOT) { + + // not use currently + } + return 0; +} +static int indexMergeCacheIntoTindex(SIndex *sIdx) { + if (sIdx == NULL) { + return -1; + } + indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + return 0; +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c new file mode 100644 index 0000000000..3c52275a4c --- /dev/null +++ b/source/libs/index/src/index_cache.c @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "index_cache.h" +#include "tcompare.h" + +#define MAX_INDEX_KEY_LEN 256// test only, change later + +// ref index_cache.h:22 +#define CACHE_KEY_LEN(p) (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) + +static char* getIndexKey(const void *pData) { + return NULL; +} +static int32_t compareKey(const void *l, const void *r) { + char *lp = (char *)l; + char *rp = (char *)r; + + // skip total len, not compare + int32_t ll, rl; // len + memcpy(&ll, lp, sizeof(int32_t)); + memcpy(&rl, rp, sizeof(int32_t)); + lp += sizeof(int32_t); + rp += sizeof(int32_t); + + // compare field id + int16_t lf, rf; // field id + memcpy(&lf, lp, sizeof(lf)); + memcpy(&rf, rp, sizeof(rf)); + if (lf != rf) { + return lf < rf ? -1: 1; + } + lp += sizeof(lf); + rp += sizeof(rf); + + // compare field type + int8_t lft, rft; + memcpy(&lft, lp, sizeof(lft)); + memcpy(&rft, rp, sizeof(rft)); + lp += sizeof(lft); + rp += sizeof(rft); + assert(rft == rft); + + // skip value len + int32_t lfl, rfl; + memcpy(&lfl, lp, sizeof(lfl)); + memcpy(&rfl, rp, sizeof(rfl)); + lp += sizeof(lfl); + rp += sizeof(rfl); + + // compare value + int32_t i, j; + for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { + if (lp[i] == rp[j]) { continue; } + else { return lp[i] < rp[j] ? -1 : 1;} + } + if (i < lfl) { return 1;} + else if (j < rfl) { return -1; } + lp += lfl; + rp += rfl; + + // skip uid + uint64_t lu, ru; + memcpy(&lu, lp, sizeof(lu)); + memcpy(&ru, rp, sizeof(ru)); + lp += sizeof(lu); + rp += sizeof(ru); + + // compare version, desc order + int32_t lv, rv; + memcpy(&lv, lp, sizeof(lv)); + memcpy(&rv, rp, sizeof(rv)); + if (lv != rv) { + return lv > rv ? -1 : 1; + } + lp += sizeof(lv); + rp += sizeof(rv); + // not care item type + + return 0; + +} +IndexCache *indexCacheCreate() { + IndexCache *cache = calloc(1, sizeof(IndexCache)); + cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); + return cache; + +} + +void indexCacheDestroy(void *cache) { + IndexCache *pCache = cache; + if (pCache == NULL) { return; } + tSkipListDestroy(pCache->skiplist); + free(pCache); +} + +int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { + if (cache == NULL) { return -1;} + + IndexCache *pCache = cache; + + // encode data + int32_t total = CACHE_KEY_LEN(term); + + char *buf = calloc(1, total); + char *p = buf; + + memcpy(p, &total, sizeof(total)); + p += sizeof(total); + + memcpy(p, &colId, sizeof(colId)); + p += sizeof(colId); + + memcpy(p, &term->colType, sizeof(term->colType)); + p += sizeof(term->colType); + + memcpy(p, &term->nColVal, sizeof(term->nColVal)); + p += sizeof(term->nColVal); + memcpy(p, term->colVal, term->nColVal); + p += term->nColVal; + + memcpy(p, &version, sizeof(version)); + p += sizeof(version); + + memcpy(p, &uid, sizeof(uid)); + p += sizeof(uid); + + memcpy(p, &term->operType, sizeof(term->operType)); + p += sizeof(term->operType); + + tSkipListPut(pCache->skiplist, (void *)buf); + return 0; + // encode end + +} +int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { + IndexCache *pCache = cache; + return 0; +} +int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { + if (cache == NULL) { return -1; } + IndexCache *pCache = cache; + SIndexTerm *term = query->term; + EIndexQueryType qtype = query->qType; + + int32_t keyLen = CACHE_KEY_LEN(term); + + char *buf = calloc(1, keyLen); + if (qtype == QUERY_TERM) { + + } else if (qtype == QUERY_PREFIX) { + + } else if (qtype == QUERY_SUFFIX) { + + } else if (qtype == QUERY_REGEX) { + + } + + return 0; +} + diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 9cb4ac6836..7aaa498864 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1330,7 +1330,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); - if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { + if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index f70b90041b..07ad45079b 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -87,9 +87,18 @@ static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { if (ssv == NULL || ctx == NULL) {return NULL;} char *data = ctx->data; + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; - return startWithStateValueCreate(Running, FST_INT, &val); + StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); + if (prefixIsMatch(ctx, nsv)) { + nsv->kind = Done; + } else { + nsv->kind = Running; + } + return nsv; } return NULL; } diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c new file mode 100644 index 0000000000..a1bba56391 --- /dev/null +++ b/source/libs/index/src/index_tfile.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "index_tfile.h" + +IndexTFile *indexTFileCreate() { + IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); + return tfile; +} +void IndexTFileDestroy(IndexTFile *tfile) { + free(tfile); +} +int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { + IndexTFile *ptfile = (IndexTFile *)tfile; + return 0; +} + + + diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index f84f874a23..6eb532b41e 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,7 +1,7 @@ add_executable(indexTest "") target_sources(indexTest PRIVATE - "indexTests.cpp" + "indexTests.cc" ) target_include_directories ( indexTest PUBLIC diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cc similarity index 74% rename from source/libs/index/test/indexTests.cpp rename to source/libs/index/test/indexTests.cc index f582536817..9dff2e9ea0 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cc @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include #include @@ -61,7 +75,7 @@ class FstReadMemory { // add later bool Search(AutomationCtx *ctx, std::vector &result) { FstStreamBuilder *sb = fstSearch(_fst, ctx); - StreamWithState *st = streamBuilderIntoStream(sb); + StreamWithState *st = streamBuilderIntoStream(sb); StreamWithStateResult *rt = NULL; while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { @@ -279,15 +293,71 @@ void validateFst() { delete m; } +class IndexEnv : public ::testing::Test { + protected: + virtual void SetUp() { + taosRemoveDir(path); + opts = indexOptsCreate(); + int ret = indexOpen(opts, path, &index); + assert(ret == 0); + } + virtual void TearDown() { + indexClose(index); + indexOptsDestroy(opts); + } + + const char *path = "/tmp/tindex"; + SIndexOpts *opts; + SIndex *index; +}; -int main(int argc, char** argv) { - checkFstPerf(); - //checkFstPrefixSearch(); - return 1; +TEST_F(IndexEnv, testPut) { + + // single index column + { + + std::string colName("tag1"), colVal("Hello world"); + SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + SIndexMultiTerm *terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + + for (size_t i = 0; i < 100; i++) { + int tableId = i; + int ret = indexPut(index, terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + // multi index column + { + + SIndexMultiTerm *terms = indexMultiTermCreate(); + { + std::string colName("tag1"), colVal("Hello world"); + SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + indexMultiTermAdd(terms, term); + } + { + std::string colName("tag2"), colVal("Hello world"); + SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); + indexMultiTermAdd(terms, term); + } + + for (int i = 0; i < 100; i++) { + int tableId = i; + int ret = indexPut(index, terms, tableId); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + } + // +} + +TEST_F(IndexEnv, testDel) { + } -//TEST(IndexFstBuilder, IndexFstInput) { -// -//} + + diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 5e635aa6a1..6ab3020935 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util common catalog function transport query + PRIVATE os util catalog function transport qcom ) ADD_SUBDIRECTORY(test) diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 954bc29e62..6ae40b0d71 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -171,8 +171,8 @@ typedef struct SCreateDbInfo { int8_t update; int8_t cachelast; SArray *keep; - int8_t dbType; - int16_t partitions; +// int8_t dbType; +// int16_t partitions; } SCreateDbInfo; typedef struct SCreateFuncInfo { diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h new file mode 100644 index 0000000000..de7cdd58b8 --- /dev/null +++ b/source/libs/parser/inc/astToMsg.h @@ -0,0 +1,11 @@ +#ifndef TDENGINE_ASTTOMSG_H +#define TDENGINE_ASTTOMSG_H + +#include "parserInt.h" +#include "taosmsg.h" + +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); +SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); +SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); + +#endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index e3925d3446..4f506e631f 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -53,6 +53,15 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t */ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen); +/** + * validate the ddl ast, and convert the ast to the corresponding message format + * @param pSqlInfo + * @param output + * @param type + * @return + */ +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); + /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. * @param pNode diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 6c95c4327b..c588a34a40 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -45,6 +45,9 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo); SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); int32_t parserValidateIdToken(SToken* pToken); +int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf); +int32_t parserValidateNameToken(SToken* pToken); + int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); @@ -63,6 +66,8 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo); STableMeta* tableMetaDup(const STableMeta* pTableMeta); +bool isDclSqlStatement(SSqlInfo* pSqlInfo); + #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 5d51293939..798c9bc97f 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -36,7 +36,6 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, i void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index); -int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index d12278632b..dda30c56fc 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -948,27 +948,24 @@ void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) { void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { pDBInfo->compressionLevel = -1; - pDBInfo->walLevel = -1; + pDBInfo->walLevel = -1; pDBInfo->fsyncPeriod = -1; - pDBInfo->commitTime = -1; + pDBInfo->commitTime = -1; pDBInfo->maxTablesPerVnode = -1; - pDBInfo->cacheBlockSize = -1; - pDBInfo->numOfBlocks = -1; + pDBInfo->cacheBlockSize = -1; + pDBInfo->numOfBlocks = -1; pDBInfo->maxRowsPerBlock = -1; pDBInfo->minRowsPerBlock = -1; - pDBInfo->daysPerFile = -1; + pDBInfo->daysPerFile = -1; pDBInfo->replica = -1; - pDBInfo->quorum = -1; - pDBInfo->keep = NULL; + pDBInfo->quorum = -1; + pDBInfo->keep = NULL; - pDBInfo->update = -1; + pDBInfo->update = -1; pDBInfo->cachelast = -1; - pDBInfo->dbType = -1; - pDBInfo->partitions = -1; - memset(&pDBInfo->precision, 0, sizeof(SToken)); } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c new file mode 100644 index 0000000000..1b46faecec --- /dev/null +++ b/source/libs/parser/src/astToMsg.c @@ -0,0 +1,160 @@ +#include "parserInt.h" +#include "parserUtil.h" + +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { + SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); + if (pMsg == NULL) { + // tscError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SUserInfo* pUser = &pInfo->pMiscInfo->user; + strncpy(pMsg->user, pUser->user.z, pUser->user.n); + pMsg->type = pUser->type; + pMsg->superUser = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + // pMsg->privilege = (char)pCmd->count; + } else { + strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + return pMsg; +} + +SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen) { + SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); + + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SToken* pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + return pShowMsg; +} + +static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid number of keep options"; + const char* msg2 = "invalid keep value"; + const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; + + pMsg->daysToKeep0 = htonl(-1); + pMsg->daysToKeep1 = htonl(-1); + pMsg->daysToKeep2 = htonl(-1); + + SArray* pKeep = pCreateDb->keep; + if (pKeep != NULL) { + size_t s = taosArrayGetSize(pKeep); +#ifdef _STORAGE + if (s >= 4 ||s <= 0) { +#else + if (s != 1) { +#endif + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + +// tListI* p0 = taosArrayGet(pKeep, 0); +// tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0; +// tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1; +// +// if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) { +// return buildInvalidOperationMsg(pMsgBuf, msg2); +// } +// if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) { +// return buildInvalidOperationMsg(pMsgBuf, msg3); +// } +// +// pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); +// pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); +// pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDbInfo, SMsgBuf* pMsgBuf) { + const char* msg = "invalid time precision"; + + pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default + + SToken* pToken = &pCreateDbInfo->precision; + if (pToken->n > 0) { + pToken->n = strdequote(pToken->z); + + if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 && + strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) { + // time precision for this db: million second + pMsg->precision = TSDB_TIME_PRECISION_MILLI; + } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 && + strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { + pMsg->precision = TSDB_TIME_PRECISION_MICRO; + } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_NANO_STR, pToken->n) == 0 && + strlen(TSDB_TIME_PRECISION_NANO_STR) == pToken->n) { + pMsg->precision = TSDB_TIME_PRECISION_NANO; + } else { + return buildInvalidOperationMsg(pMsgBuf, msg); + } + } + + return TSDB_CODE_SUCCESS; +} + +static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { + pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); + pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); + pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); + pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); + pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); + pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); + pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); + pMsg->compression = pCreateDb->compressionLevel; + pMsg->walLevel = (char)pCreateDb->walLevel; + pMsg->replications = pCreateDb->replica; + pMsg->quorum = pCreateDb->quorum; + pMsg->ignoreExist = pCreateDb->ignoreExists; + pMsg->update = pCreateDb->update; + pMsg->cacheLastRow = pCreateDb->cachelast; +} + +int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { + doSetDbOptions(pCreateDbMsg, pCreateDbSql); + + if (setKeepOption(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (setTimePrecision(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // todo configurable + pCreateDbMsg->numOfVgroups = htonl(2); + + return TSDB_CODE_SUCCESS; +} + +SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen) { + SCreateDbMsg* pCreateMsg = calloc(1, sizeof(SCreateDbMsg)); + + SMsgBuf msg = {.buf = msgBuf, .len = msgLen}; + if (setDbOptions(pCreateMsg, pCreateDbInfo, &msg) != TSDB_CODE_SUCCESS) { + tfree(pCreateMsg); + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + + return NULL; + } + + return pCreateMsg; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 9f1363a9b4..135774cd3b 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "astGenerator.h" #include "function.h" #include "parserInt.h" @@ -23,6 +22,7 @@ #include "tglobal.h" #include "tmsgtype.h" #include "ttime.h" +#include "astToMsg.h" #define TSQL_TBNAME_L "tbname" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -759,11 +759,6 @@ int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMs // It is a time window query pQueryInfo->info.timewindow = true; return TSDB_CODE_SUCCESS; - // disable it temporarily -// bool interpQuery = tscIsPointInterpQuery(pQueryInfo); -// if ((pSqlNode->interval.token == TK_EVERY && (!interpQuery)) || (pSqlNode->interval.token == TK_INTERVAL && interpQuery)) { -// return buildInvalidOperationMsg(pMsgBuf, msg4); -// } } int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSession, int32_t precision, SMsgBuf* pMsgBuf) { @@ -3636,11 +3631,14 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) } int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msgBuf, int32_t msgBufLen) { - //1. if it is a query, get the meta info and continue. assert(pCatalog != NULL && pInfo != NULL); int32_t code = 0; -#if 0 + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf *pMsgBuf = &m; + switch (pInfo->type) { +#if 0 case TSDB_SQL_DROP_TABLE: case TSDB_SQL_DROP_USER: case TSDB_SQL_DROP_ACCT: @@ -3651,14 +3649,14 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (parserValidateIdToken(pzName) != TSDB_CODE_SUCCESS)) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } if (pInfo->type == TSDB_SQL_DROP_DB) { assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { @@ -3675,7 +3673,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer strncpy(pCmd->payload, pzName->z, pzName->n); } else { // drop user/account if (pzName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } strncpy(pCmd->payload, pzName->z, pzName->n); @@ -3689,12 +3687,12 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); if (ret != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } break; @@ -3704,14 +3702,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer return TSDB_CODE_SUCCESS; } - case TSDB_SQL_SHOW: { - if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - case TSDB_SQL_CREATE_FUNCTION: case TSDB_SQL_DROP_FUNCTION: { code = handleUserDefinedFunc(pSql, pInfo); @@ -3722,40 +3712,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer break; } - case TSDB_SQL_ALTER_DB: - case TSDB_SQL_CREATE_DB: { - const char* msg1 = "invalid db name"; - const char* msg2 = "name too long"; - - SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); - if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg2); - } - - char buf[TSDB_DB_NAME_LEN] = {0}; - SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); - - if (tscValidateName(&token) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token); - if (ret != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); - } - - if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - case TSDB_SQL_CREATE_DNODE: { const char* msg = "invalid host name (ip address)"; if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -3779,11 +3740,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } if (pName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; @@ -3793,7 +3754,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { } else { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } } @@ -3805,7 +3766,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } // additional msg has been attached already code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); @@ -3821,7 +3782,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); @@ -3836,11 +3797,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } if (pToken->n > TSDB_DB_NAME_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); } @@ -3853,7 +3814,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer /* validate the parameter names and options */ if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } char* pMsg = pCmd->payload; @@ -3867,7 +3828,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer strncpy(pCfg->ep, t0->z, t0->n); if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } strncpy(pCfg->config, t1->z, t1->n); @@ -3882,65 +3843,13 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer break; } - case TSDB_SQL_CREATE_USER: - case TSDB_SQL_ALTER_USER: { - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - const char* msg5 = "invalid user rights"; - const char* msg7 = "not support options"; - - pCmd->command = pInfo->type; - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - SToken* pName = &pUser->user; - SToken* pPwd = &pUser->passwd; - - if (pName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); - } - - if (pCmd->command == TSDB_SQL_CREATE_USER) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pUser->type == TSDB_ALTER_USER_PASSWD) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - assert(pPwd->type == TSDB_DATA_TYPE_NULL); - - SToken* pPrivilege = &pUser->privilege; - - if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 1; - } else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) { - pCmd->count = 2; - } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 3; - } else { - return setInvalidOperatorMsg(pMsgBuf, msg5); - } - } else { - return setInvalidOperatorMsg(pMsgBuf, msg7); - } - } - - break; - } - case TSDB_SQL_CFG_LOCAL: { SMiscInfo *pMiscInfo = pInfo->pMiscInfo; const char *msg = "invalid configure options or values"; // validate the parameter names and options if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); @@ -3995,7 +3904,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } // normalizeSqlNode(pSqlNode); // normalize the column name in each function @@ -4061,21 +3970,22 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } break; } case TSDB_SQL_COMPACT_VNODE:{ const char* msg = "invalid compact"; if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } break; } + #endif default: - return setInvalidOperatorMsg(pMsgBuf, "not support sql expression"); + return buildInvalidOperationMsg(pMsgBuf, "not support sql expression"); } -#endif + SCatalogReq req = {0}; SMetaData data = {0}; @@ -4114,5 +4024,247 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } + return code; +} + +// todo remove it +static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLen, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid name"; + const char* msg2 = "wildcard string should be less than %d characters"; + const char* msg3 = "database name too long"; + const char* msg4 = "pattern is invalid"; + const char* msg5 = "database name is empty"; + const char* msg6 = "pattern string is empty"; + + /* + * database prefix in pInfo->pMiscInfo->a[0] + * wildcard in like clause in pInfo->pMiscInfo->a[1] + */ + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; + int16_t showType = pShowInfo->showType; + if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_VGROUP) { + SToken* pDbPrefixToken = &pShowInfo->prefix; + if (pDbPrefixToken->type != 0) { + if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (pDbPrefixToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); + // if (ret != TSDB_CODE_SUCCESS) { + // return buildInvalidOperationMsg(pMsgBuf, msg1); + // } + } + + // show table/stable like 'xxxx', set the like pattern for show tables + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type != 0) { + if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + pPattern->n = strdequote(pPattern->z); + if (pPattern->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + if (pPattern->n > tsMaxWildCardsLen) { + char tmp[64] = {0}; + sprintf(tmp, msg2, tsMaxWildCardsLen); + return buildInvalidOperationMsg(pMsgBuf, tmp); + } + } + } else if (showType == TSDB_MGMT_TABLE_VNODES) { + if (pShowInfo->prefix.type == 0) { + return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); + } + + if (pShowInfo->prefix.type == TK_STRING) { + pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); + } + } + + *output = buildShowMsg(pShowInfo, 0, pMsgBuf->buf, pMsgBuf->len); + *msgLen = sizeof(SShowMsg)/* + htons(pShowMsg->payloadLen)*/; + return TSDB_CODE_SUCCESS; +} + +// can only perform the parameters based on the macro definitation +static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { + char msg[512] = {0}; + + if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->replications != -1 && + (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t blocks = ntohl(pCreate->totalBlocks); + if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, + TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->quorum != -1 && + (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, + TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t val = htonl(pCreate->daysPerFile); + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { + snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, + TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = htonl(pCreate->cacheBlockSize); + if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { + snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, + TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && + pCreate->precision != TSDB_TIME_PRECISION_NANO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = htonl(pCreate->commitTime); + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { + snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, + TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = htonl(pCreate->fsyncPeriod); + if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { + snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, + TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->compression != -1 && + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { + int32_t code = 0; + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf *pMsgBuf = &m; + + *type = pInfo->type; + + switch (pInfo->type) { + case TSDB_SQL_CREATE_USER: + case TSDB_SQL_ALTER_USER: { + const char* msg1 = "not support options"; + const char* msg2 = "invalid user/account name"; + const char* msg3 = "name too long"; + const char* msg4 = "invalid user rights"; + + SUserInfo* pUser = &pInfo->pMiscInfo->user; + SToken* pName = &pUser->user; + SToken* pPwd = &pUser->passwd; + + if (pName->n >= TSDB_USER_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + if (pInfo->type == TSDB_SQL_CREATE_USER) { + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } else { + if (pUser->type == TSDB_ALTER_USER_PASSWD) { + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + assert(pPwd->type == TSDB_DATA_TYPE_NULL); + + SToken* pPrivilege = &pUser->privilege; + if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { + // pCmd->count = 1; + } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) { + // pCmd->count = 2; + } else { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + } else { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_SHOW: { + code = setShowInfo(pInfo, output, outputLen, pMsgBuf); + break; + } + + case TSDB_SQL_ALTER_DB: + case TSDB_SQL_CREATE_DB: { + const char* msg1 = "invalid db name"; + const char* msg2 = "name too long"; + + SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); + if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + char buf[TSDB_DB_NAME_LEN] = {0}; + SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); + + if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pMsgBuf->buf, pMsgBuf->len); + if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + strncpy(pCreateMsg->db, token.z, token.n); + + *output = pCreateMsg; + *outputLen = sizeof(SCreateDbMsg); + break; + } + + default: + break; + } + return code; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index f63f13d8ac..877ecb2c85 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -77,6 +77,14 @@ typedef struct SInsertParseContext { SInsertStmtInfo* pOutput; } SInsertParseContext; +typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param); + +typedef struct SKvParam { + char buf[TSDB_MAX_TAGS_LEN]; + SKVRowBuilder* builder; + SSchema* schema; +} SKvParam; + static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; @@ -300,14 +308,6 @@ static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec return TSDB_CODE_SUCCESS; } -typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param); - -typedef struct SKvParam { - char buf[TSDB_MAX_TAGS_LEN]; - SKVRowBuilder* builder; - SSchema* schema; -} SKvParam; - static FORCE_INLINE int32_t KvRowAppend(const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*)param; if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d167f7ad3c..e0ac7c295b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,25 +31,36 @@ bool qIsInsertSql(const char* pStr, size_t length) { } while (1); } -int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) { - *pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (*pQueryInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. - } - +int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + return terrno; } - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(NULL, &pCatalog); - if (code) { - return code; + if (isDclSqlStatement(&info)) { + int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen); + if (code == TSDB_CODE_SUCCESS) { + // do nothing + } + } else { + SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. + return terrno; + } + + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(NULL, &pCatalog); + code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); + if (code == TSDB_CODE_SUCCESS) { + *pOutput = pQueryInfo; + } } - - return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); + + destroySqlInfo(&info); + return TSDB_CODE_SUCCESS; } int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { @@ -66,7 +77,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) { SName* pn1 = (SName*)p1; SName* pn2 = (SName*)p2; - int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId)); + int32_t ret = pn1->acctId - pn2->acctId; if (ret != 0) { return ret > 0? 1:-1; } else { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7fcb7ea304..b72bc06324 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "parserUtil.h" + #include "taosmsg.h" #include "parser.h" @@ -22,6 +22,8 @@ #include "thash.h" #include "tbuffer.h" #include "parserInt.h" +#include "parserUtil.h" +#include "tmsgtype.h" #include "queryInfoUtil.h" #include "function.h" @@ -97,12 +99,54 @@ int32_t parserValidateIdToken(SToken* pToken) { return TSDB_CODE_SUCCESS; } +int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) { + const char* msg1 = "password can not be empty"; + const char* msg2 = "name or password too long"; + const char* msg3 = "password needs single quote marks enclosed"; + + if (pToken->type != TK_STRING) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + strdequote(pToken->z); + + pToken->n = (uint32_t)strtrim(pToken->z); // trim space before and after passwords + if (pToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + if (pToken->n >= TSDB_USET_PASSWORD_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t parserValidateNameToken(SToken* pToken) { + if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // it is a token quoted with escape char '`' + if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) { + return TSDB_CODE_SUCCESS; + } + + char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); + if (sep != NULL) { // It is a complex type, not allow + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + strntolower(pToken->z, pToken->z, pToken->n); + return TSDB_CODE_SUCCESS; +} + int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { strncpy(pBuf->buf, msg, pBuf->len); return TSDB_CODE_TSC_INVALID_OPERATION; } -int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; const char* msgFormat3 = "%s"; @@ -584,21 +628,6 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } -int32_t getFirstInvisibleFieldPos(SQueryStmtInfo* pQueryInfo) { - if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) { - return 0; - } - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (!pField->visible) { - return i; - } - } - - return pQueryInfo->fieldsInfo.numOfOutput; -} - SInternalField* appendFieldInfo(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { assert(pFieldInfo != NULL); pFieldInfo->numOfOutput++; @@ -1583,6 +1612,10 @@ uint32_t convertRelationalOperator(SToken *pToken) { } } +bool isDclSqlStatement(SSqlInfo* pSqlInfo) { + return (pSqlInfo->type != TSDB_SQL_SELECT); +} + #if 0 int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) { memset(pQueryAttr, 0, sizeof(SQueryAttr)); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 53a7ba48f8..d674462fc0 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2519,7 +2519,7 @@ static void yy_reduce( { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; case 105: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;} +{setDefaultCreateDbOption(&yymsp[1].minor.yy256);} break; case 106: /* db_optr ::= db_optr cache */ { yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } @@ -2590,16 +2590,16 @@ static void yy_reduce( break; case 121: /* topic_optr ::= db_optr */ case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131); -{ yylhsminor.yy256 = yymsp[0].minor.yy256; yylhsminor.yy256.dbType = TSDB_DB_TYPE_TOPIC; } +{ yylhsminor.yy256 = yymsp[0].minor.yy256;} yymsp[0].minor.yy256 = yylhsminor.yy256; break; case 122: /* topic_optr ::= topic_optr partitions */ case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132); -{ yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yylhsminor.yy256 = yymsp[-1].minor.yy256; } yymsp[-1].minor.yy256 = yylhsminor.yy256; break; case 123: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;} +{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); } break; case 133: /* typename ::= ids */ { diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 03b76152da..feae008273 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -15,7 +15,7 @@ TARGET_INCLUDE_DIRECTORIES( TARGET_LINK_LIBRARIES( parserTest - PUBLIC os util common parser catalog transport gtest function planner query + PUBLIC os util common parser catalog transport gtest function planner qcom ) TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 5877adf41c..85db46d7bf 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -57,17 +57,20 @@ void *__wrap_malloc(size_t c) { // [...]; class InsertTest : public Test { protected: - void setDatabase(const string& db) { + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; db_ = db; } void bind(const char* sql) { reset(); - cxt_.sqlLen = strlen(sql); + cxt_.pAcctId = acctId_.c_str(); + cxt_.pDbname = db_.c_str(); strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); sqlBuf_[cxt_.sqlLen] = '\0'; cxt_.pSql = sqlBuf_; - cxt_.pDbname = db_.c_str(); + } int32_t run() { @@ -95,6 +98,7 @@ private: res_ = nullptr; } + string acctId_; string db_; char errMagBuf_[max_err_len]; char sqlBuf_[max_sql_len]; @@ -105,7 +109,7 @@ private: // INSERT INTO tb_name VALUES (field1_value, ...) TEST_F(InsertTest, simpleTest) { - setDatabase("test"); + setDatabase("root", "test"); bind("insert into t1 values (now, 1, \"wxy\")"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); @@ -116,7 +120,7 @@ TEST_F(InsertTest, simpleTest) { } TEST_F(InsertTest, toleranceTest) { - setDatabase("test"); + setDatabase("root", "test"); bind("insert into"); ASSERT_NE(run(), TSDB_CODE_SUCCESS); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index f2b34971ef..4e6b94081b 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -20,34 +20,42 @@ namespace { void generateTestT1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); builder.done(); } void generateTestST1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); builder.done(); - mcs->createSubTable("test", "st1", "st1s1", 1); - mcs->createSubTable("test", "st1", "st1s2", 2); + mcs->createSubTable("root.test", "st1", "st1s1", 1); + mcs->createSubTable("root.test", "st1", "st1s2", 2); } } -void generateMetaData(MockCatalogService* mcs) { - generateTestT1(mcs); - generateTestST1(mcs); - mcs->showTables(); +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + return mockCatalogService->catalogGetHandle(clusterId, catalogHandle); } -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { - return mockCatalogService->getCatalogHandle(pMgmtEps); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return mockCatalogService->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) { - return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +void initMetaDataEnv() { + mockCatalogService.reset(new MockCatalogService()); +} + +void generateMetaData() { + generateTestT1(mockCatalogService.get()); + generateTestST1(mockCatalogService.get()); + mockCatalogService->showTables(); +} + +void destroyMetaDataEnv() { + mockCatalogService.reset(); } diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index e60f7727e6..c9812d5b94 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -18,10 +18,12 @@ #include "mockCatalogService.h" -void generateMetaData(MockCatalogService* mcs); +void initMetaDataEnv(); +void generateMetaData(); +void destroyMetaDataEnv(); // mock -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 457f8d88bd..9bc029cbd8 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -27,27 +27,28 @@ std::unique_ptr mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { - assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns); - SSchema* col = meta_->schema + colIndex_; + assert(colId_ < schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); + SSchema* col = schema()->schema + colId_; col->type = type; - col->colId = colIndex_++; + col->colId = colId_++; col->bytes = bytes; strcpy(col->name, name.c_str()); return *this; } virtual TableBuilder& setVgid(int16_t vgid) { - meta_->vgId = vgid; + schema()->vgId = vgid; + meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); return *this; } virtual TableBuilder& setPrecision(uint8_t precision) { - meta_->tableInfo.precision = precision; + schema()->tableInfo.precision = precision; return *this; } virtual void done() { - meta_->tableInfo.rowSize = rowsize_; + schema()->tableInfo.rowSize = rowsize_; } private: @@ -64,61 +65,62 @@ private: return std::unique_ptr(new TableBuilder(meta)); } - TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) { + TableBuilder(STableMeta* schemaMeta) : colId_(0), rowsize_(0), meta_(new MockTableMeta()) { + meta_->schema.reset(schemaMeta); } - STableMeta* table() { + std::shared_ptr schema() { + return meta_->schema; + } + + std::shared_ptr table() { return meta_; } - int32_t colIndex_; + int32_t colId_; int32_t rowsize_; - STableMeta* meta_; + std::shared_ptr meta_; }; class MockCatalogServiceImpl { public: static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogServiceImpl() { + MockCatalogServiceImpl() : id_(1) { } - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const { - return (struct SCatalog*)0x01; + int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + return 0; } - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { - assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); - SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; - int32_t code = copyTableMeta(fullName->dbname, fullName->tname, &table); + int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table); if (TSDB_CODE_SUCCESS != code) { return code; } - std::unique_ptr tables((SArray*)taosArrayInit(1, sizeof(STableMeta*))); - if (!tables) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - STableMeta* elem = table.release(); - taosArrayPush(tables.get(), &elem); - pMetaData->pTableMeta = tables.release(); + *pTableMeta = table.release(); return TSDB_CODE_SUCCESS; } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); - meta_[db][tbname].reset(builder_->table()); - meta_[db][tbname]->uid = id_++; + meta_[db][tbname] = builder_->table(); + meta_[db][tbname]->schema->uid = id_++; return *(builder_.get()); } void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { std::unique_ptr table; - if (TSDB_CODE_SUCCESS != copyTableMeta(db, stbname, &table)) { - throw std::runtime_error("copyTableMeta failed"); + if (TSDB_CODE_SUCCESS != copyTableSchemaMeta(db, stbname, &table)) { + throw std::runtime_error("copyTableSchemaMeta failed"); } - meta_[db][tbname].reset(table.release()); - meta_[db][tbname]->uid = id_++; + meta_[db][tbname].reset(new MockTableMeta()); + meta_[db][tbname]->schema.reset(table.release()); + meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + // super table + meta_[db][stbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); } void showTables() const { @@ -148,29 +150,43 @@ public: std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { - std::cout << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + const auto& schema = table.second->schema; + std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << std::endl; } std::cout << std::endl; } for (const auto& db : meta_) { for (const auto& table : db.second) { + const auto& schema = table.second->schema; std::cout << "Table:" << table.first << std::endl; std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; std::cout << SL(3, 1) << std::endl; - int16_t numOfTags = table.second->tableInfo.numOfTags; - int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; + int16_t numOfTags = schema->tableInfo.numOfTags; + int16_t numOfFields = numOfTags + schema->tableInfo.numOfColumns; for (int16_t i = 0; i < numOfFields; ++i) { - const SSchema* schema = table.second->schema + i; - std::cout << SF(std::string(schema->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + const SSchema* col = schema->schema + i; + std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; } std::cout << std::endl; } } } + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + DbMetaCache::const_iterator it = meta_.find(db); + if (meta_.end() == it) { + return std::shared_ptr(); + } + TableMetaCache::const_iterator tit = it->second.find(tbname); + if (it->second.end() == tit) { + return std::shared_ptr(); + } + return tit->second; + } + private: - typedef std::map > TableMetaCache; + typedef std::map> TableMetaCache; typedef std::map DbMetaCache; std::string ttToString(int8_t tableType) const { @@ -207,20 +223,13 @@ private: return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); } - std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { - DbMetaCache::const_iterator it = meta_.find(db); - if (meta_.end() == it) { - return std::shared_ptr(); - } - TableMetaCache::const_iterator tit = it->second.find(tbname); - if (it->second.end() == tit) { - return std::shared_ptr(); - } - return tit->second; + std::shared_ptr getTableSchemaMeta(const std::string& db, const std::string& tbname) const { + std::shared_ptr table = getTableMeta(db, tbname); + return table ? table->schema : std::shared_ptr(); } - int32_t copyTableMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { - std::shared_ptr src = getTableMeta(db, tbname); + int32_t copyTableSchemaMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { + std::shared_ptr src = getTableSchemaMeta(db, tbname); if (!src) { return TSDB_CODE_TSC_INVALID_TABLE_NAME; } @@ -244,14 +253,6 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() { } -struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) const { - return impl_->getCatalogHandle(pMgmtEps); -} - -int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { - return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); -} - ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } @@ -262,4 +263,16 @@ void MockCatalogService::createSubTable(const std::string& db, const std::string void MockCatalogService::showTables() const { impl_->showTables(); -} \ No newline at end of file +} + +std::shared_ptr MockCatalogService::getTableMeta(const std::string& db, const std::string& tbname) const { + return impl_->getTableMeta(db, tbname); +} + +int32_t MockCatalogService::catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + return impl_->catalogGetHandle(clusterId, catalogHandle); +} + +int32_t MockCatalogService::catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + return impl_->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); +} diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 66b439b3e9..cd01db09cc 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -18,6 +18,7 @@ #include #include +#include #include "catalog.h" @@ -41,19 +42,24 @@ public: virtual void done() = 0; }; -class MockCatalogServiceImpl; +struct MockTableMeta { + std::shared_ptr schema; + std::vector vgs; +}; +class MockCatalogServiceImpl; class MockCatalogService { public: - static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogService(); ~MockCatalogService(); - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const; ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const; + + // mock interface + int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const; + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; private: std::unique_ptr impl_; diff --git a/source/libs/parser/test/parserMain.cpp b/source/libs/parser/test/parserTestMain.cpp similarity index 88% rename from source/libs/parser/test/parserMain.cpp rename to source/libs/parser/test/parserTestMain.cpp index 5ec304a006..7de2cb66c2 100644 --- a/source/libs/parser/test/parserMain.cpp +++ b/source/libs/parser/test/parserTestMain.cpp @@ -22,12 +22,12 @@ class ParserEnv : public testing::Environment { public: virtual void SetUp() { - mockCatalogService.reset(new MockCatalogService()); - generateMetaData(mockCatalogService.get()); + initMetaDataEnv(); + generateMetaData(); } virtual void TearDown() { - mockCatalogService.reset(); + destroyMetaDataEnv(); } ParserEnv() {} diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 6a402cd620..396e8a12fe 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -29,6 +29,7 @@ #include "taos.h" #include "tdef.h" #include "tvariant.h" +#include "parserUtil.h" namespace { void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) { @@ -700,5 +701,47 @@ TEST(testCase, function_Test6) { destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, show_user_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + char sql1[] = "show users"; + SSqlInfo info1 = doGenerateAST(sql1); + ASSERT_EQ(info1.valid, true); + + void* output = NULL; + int32_t type = 0; + int32_t len = 0; + int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + ASSERT_EQ(code, 0); + + // convert the show command to be the select query + // select name, privilege, create_time, account from information_schema.users; +} + +TEST(testCase, create_user_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + char sql[] = {"create user abc pass 'abc'"}; + + SSqlInfo info1 = doGenerateAST(sql); + ASSERT_EQ(info1.valid, true); + + ASSERT_EQ(isDclSqlStatement(&info1), true); + + void* output = NULL; + int32_t type = 0; + int32_t len = 0; + int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + ASSERT_EQ(code, 0); + destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 4e0d03d07a..7f8c118663 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util common catalog parser transport function query + PRIVATE os util catalog cjson parser transport function qcom ) ADD_SUBDIRECTORY(test) diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 6c65e4810d..b1c4643f15 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -99,19 +99,16 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); */ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); -/** - * Convert to physical plan to string to enable to print it out in the shell. - * @param pPhyNode - * @param str - * @return - */ -int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str); +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); +int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); +int32_t subPlanToString(const SSubplan *pPhyNode, char** str); +int32_t stringToSubplan(const char* str, SSubplan** subplan); /** * Destroy the query plan object. * @return */ -void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode); +void destroyQueryPlan(struct SQueryPlanNode* pQueryNode); /** * Destroy the physical plan. @@ -120,6 +117,9 @@ void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode); */ void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode); +const char* opTypeToOpName(int32_t type); +int32_t opNameToOpType(const char* name); + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c new file mode 100644 index 0000000000..ce923314bd --- /dev/null +++ b/source/libs/planner/src/logicPlan.c @@ -0,0 +1,604 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "function.h" +#include "os.h" +#include "parser.h" +#include "plannerInt.h" + +typedef struct SFillEssInfo { + int32_t fillType; // fill type + int64_t *val; // fill value +} SFillEssInfo; + +typedef struct SJoinCond { + bool tagExists; // denote if tag condition exists or not + SColumn *tagCond[2]; + SColumn *colCond[2]; +} SJoinCond; + +static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); +static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); + +int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len); +int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { + return 0; +} + +int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { + SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); + assert(taosArrayGetSize(upstream) == 1); + + *pQueryNode = taosArrayGetP(upstream, 0); + + taosArrayDestroy(upstream); + return TSDB_CODE_SUCCESS; +} + +int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { + return 0; +} + +void destroyQueryPlan(SQueryPlanNode* pQueryNode) { + if (pQueryNode == NULL) { + return; + } + + doDestroyQueryNode(pQueryNode); +} + +//====================================================================================================================== + +static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, + SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { + SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); + + pNode->info.type = type; + pNode->info.name = strdup(name); + + pNode->numOfExpr = numOfOutput; + pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); + + for(int32_t i = 0; i < numOfOutput; ++i) { + taosArrayPush(pNode->pExpr, &pExpr[i]); + } + + pNode->pChildren = taosArrayInit(4, POINTER_BYTES); + for(int32_t i = 0; i < numOfPrev; ++i) { + taosArrayPush(pNode->pChildren, &prev[i]); + } + + switch(type) { + case QNODE_TAGSCAN: + case QNODE_TABLESCAN: { + SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); + memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); + info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); + pNode->pExtInfo = info; + break; + } + + case QNODE_TIMEWINDOW: { + SInterval* pInterval = calloc(1, sizeof(SInterval)); + pNode->pExtInfo = pInterval; + memcpy(pInterval, pExtInfo, sizeof(SInterval)); + break; + } + + case QNODE_STATEWINDOW: { + SColumn* psw = calloc(1, sizeof(SColumn)); + pNode->pExtInfo = psw; + memcpy(psw, pExtInfo, sizeof(SColumn)); + break; + } + + case QNODE_SESSIONWINDOW: { + SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); + pNode->pExtInfo = pSessionWindow; + memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); + break; + } + + case QNODE_GROUPBY: { + SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; + + SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); + pGroupbyExpr->groupbyTag = p->groupbyTag; + pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); + + pNode->pExtInfo = pGroupbyExpr; + break; + } + + case QNODE_FILL: { // todo !! + pNode->pExtInfo = pExtInfo; + break; + } + + case QNODE_LIMIT: { + pNode->pExtInfo = calloc(1, sizeof(SLimit)); + memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); + break; + } + + case QNODE_SORT: { + pNode->pExtInfo = taosArrayDup(pExtInfo); + break; + } + + default: + break; + } + + return pNode; +} + +static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, + SArray* pExprs, SArray* tableCols) { + if (pQueryInfo->info.onlyTagQuery) { + int32_t num = (int32_t) taosArrayGetSize(pExprs); + SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info); + + if (pQueryInfo->info.distinct) { + pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); + } + + return pNode; + } + + SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); + + if (pQueryInfo->info.projectionQuery) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); + } else { + STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); + + // table source column projection, generate the projection expr + int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); + SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(tableCols, i); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, pCol); + SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); + SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); + pExpr[i] = p; + } + + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); + tfree(pExpr); + } + + return pNode; +} + +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { + // group by column not by tag + size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + + // check for aggregation + int32_t level = getExprFunctionLevel(pQueryInfo); + + for(int32_t i = level - 1; i >= 0; --i) { + SArray* p = pQueryInfo->exprList[i]; + size_t num = taosArrayGetSize(p); + + bool aggregateFunc = false; + for(int32_t j = 0; j < num; ++j) { + SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); + if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { + continue; + } + + aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); + if (aggregateFunc) { + break; + } + } + + if (aggregateFunc) { + if (pQueryInfo->interval.interval > 0) { + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); + } else if (pQueryInfo->sessionWindow.gap > 0) { + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); + } else if (pQueryInfo->stateWindow.col.info.colId > 0) { + pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); + } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); + } else { + pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); + } + } else { + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); + } + } + + if (pQueryInfo->havingFieldNum > 0) { +// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); +// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); + } + + if (pQueryInfo->fillType != TSDB_FILL_NONE) { + SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); + pInfo->fillType = pQueryInfo->fillType; + pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); + memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); + + SArray* p = pQueryInfo->exprList[0]; // top expression in select clause + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); + } + + if (pQueryInfo->order != NULL) { + SArray* pList = pQueryInfo->exprList[0]; + pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); + } + + if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { + pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); + } + + return pNode; +} + +static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, + SArray* tableCols) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); + + SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; + + // handle the only tag query + SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); + if (pQueryInfo->info.onlyTagQuery) { + tfree(info.tableName); + return pNode; + } + + SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); + tfree(info.tableName); + return pNode1; +} + +static bool isAllAggExpr(SArray* pList) { + assert(pList != NULL); + + for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { + SExprInfo* p = taosArrayGetP(pList, k); + if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { + return false; + } + } + + return true; +} + +SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { + SArray* upstream = NULL; + + if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause + upstream = taosArrayInit(4, POINTER_BYTES); + + size_t size = taosArrayGetSize(pQueryInfo->pUpstream); + for(int32_t i = 0; i < size; ++i) { + SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); + SArray* p = createQueryPlanImpl(pq); + taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); + } + } + + if (pQueryInfo->numOfTables > 1) { // it is a join query + // 1. separate the select clause according to table + taosArrayDestroy(upstream); + upstream = taosArrayInit(5, POINTER_BYTES); + + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; + uint64_t uid = pTableMetaInfo->pTableMeta->uid; + + SArray* exprList = taosArrayInit(4, POINTER_BYTES); + if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; +// dropAllExprInfo(exprList); + exit(-1); + } + + // 2. create the query execution node + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; + + // 3. get the required table column list + SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); + columnListCopy(tableColumnList, pQueryInfo->colList, uid); + + // 4. add the projection query node + SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); + columnListDestroy(tableColumnList); +// dropAllExprInfo(exprList); + taosArrayPush(upstream, &pNode); + } + + // 3. add the join node here + SQueryTableInfo info = {0}; + int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); + SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, + pQueryInfo->exprList[0]->pData, num, NULL); + + // 4. add the aggregation or projection execution node + pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } else { // only one table, normal query process + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } + + return upstream; +} + +static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { + tfree(pQueryNode->pExtInfo); + tfree(pQueryNode->pSchema); + tfree(pQueryNode->info.name); +// dropAllExprInfo(pQueryNode->pExpr); + + if (pQueryNode->pChildren != NULL) { + int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i); + doDestroyQueryNode(p); + } + + taosArrayDestroy(pQueryNode->pChildren); + } + + tfree(pQueryNode); +} + +static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { + if (level > 0) { + sprintf(buf + totalLen, "%*c", level, ' '); + totalLen += level; + } + + int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); + int32_t len = len1 + totalLen; + + switch(pQueryNode->info.type) { + case QNODE_TABLESCAN: { + SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, + pInfo->window.skey, pInfo->window.ekey); + assert(len1 > 0); + len += len1; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); + len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); + + assert(len1 > 0); + len += len1; + } + + len1 = sprintf(buf + len, "\n"); + assert(len1 > 0); + + len += len1; + break; + } + + case QNODE_PROJECT: { + len1 = sprintf(buf + len, "cols:"); + assert(len1 > 0); + len += len1; + + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ")"); + len += len1; + + // todo print filter info + len1 = sprintf(buf + len, " filters:(nil)\n"); + len += len1; + break; + } + + case QNODE_AGGREGATE: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ")\n"); + len += len1; + + break; + } + + case QNODE_TIMEWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ") "); + len += len1; + + SInterval* pInterval = pQueryNode->pExtInfo; + + // todo dynamic return the time precision + len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", + pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, + TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); + len += len1; + + break; + } + + case QNODE_STATEWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ") "); + len += len1; + + SColumn* pCol = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); + len += len1; + break; + } + + case QNODE_SESSIONWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + + len1 = sprintf(buf + len, ") "); + len += len1; + + struct SSessionWindow* ps = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); + len += len1; + break; + } + + case QNODE_GROUPBY: { + len = printExprInfo(buf, pQueryNode, len); + + SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, ") groupby_col: "); + len += len1; + + for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { + SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); + len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); + len += len1; + } + + len += sprintf(buf + len, "\n"); + break; + } + + case QNODE_FILL: { + SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%d", pEssInfo->fillType); + len += len1; + + if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { + len1 = sprintf(buf + len, ", val:"); + len += len1; + + // todo get the correct fill data type + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); + len += len1; + + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_LIMIT: { + SLimit* pVal = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); + len += len1; + break; + } + + case QNODE_DISTINCT: + case QNODE_TAGSCAN: { + len1 = sprintf(buf + len, "cols: "); + len += len1; + + len = printExprInfo(buf, pQueryNode, len); + + len1 = sprintf(buf + len, ")\n"); + len += len1; + + break; + } + + case QNODE_SORT: { + len1 = sprintf(buf + len, "cols:"); + len += len1; + + SArray* pSort = pQueryNode->pExtInfo; + for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { + SOrder* p = taosArrayGet(pSort, i); + len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); + + len += len1; + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_JOIN: { + // print join condition + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + } + + return len; +} + +int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { + int32_t len1 = 0; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); + + SSqlExpr* pExpr = &pExprInfo->base; + len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + assert(len1 > 0); + + len += len1; + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + + return len; +} + +int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { + int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); + + for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) { + SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i); + int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); + len = len1; + } + + return len; +} + +int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { + assert(pQueryNode); + + *str = calloc(1, 4096); + + int32_t len = sprintf(*str, "===== logic plan =====\n"); + queryPlanToStringImpl(*str, pQueryNode, 0, len); + + return TSDB_CODE_SUCCESS; +} + +SQueryPlanNode* queryPlanFromString() { + return NULL; +} diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index e7acb12bc0..1d29e48e30 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -16,12 +16,8 @@ #include "plannerInt.h" #include "parser.h" -static const char* gOpName[] = { - "Unknown", -#define INCLUDE_AS_NAME -#include "plannerOp.h" -#undef INCLUDE_AS_NAME -}; +#define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan +#define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _ typedef struct SPlanContext { struct SCatalog* pCatalog; @@ -30,21 +26,115 @@ typedef struct SPlanContext { SSubplanId nextId; } SPlanContext; -static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); +static const char* gOpName[] = { + "Unknown", +#define INCLUDE_AS_NAME +#include "plannerOp.h" +#undef INCLUDE_AS_NAME +}; + +const char* opTypeToOpName(int32_t type) { + return gOpName[type]; +} + +int32_t opNameToOpType(const char* name) { + for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { + if (strcmp(name, gOpName[i])) { + return i; + } + } + return OP_Unknown; +} + +static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfCols; + dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); + if (NULL == dataBlockSchema->pSchema) { + return false; + } + memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + return true; +} + +static bool cloneExprArray(SArray** dst, SArray* src) { + if (NULL == src) { + return true; + } + size_t size = taosArrayGetSize(src); + if (0 == size) { + return true; + } + *dst = taosArrayInit(size, POINTER_BYTES); + if (NULL == *dst) { + return false; + } + return (TSDB_CODE_SUCCESS == copyAllExprInfo(*dst, src, true) ? true : false); } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { SPhyNode* node = (SPhyNode*)calloc(1, size); + if (NULL == node) { + return NULL; + } node->info.type = type; - node->info.name = gOpName[type]; - SWAP(node->pTargets, pPlanNode->pExpr, SArray*); - toDataBlockSchema(pPlanNode, &(node->targetSchema)); + node->info.name = opTypeToOpName(type); + if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { + free(node); + return NULL; + } + return node; +} + +static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { + SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size); + node->uid = pTable->pMeta->pTableMeta->uid; + node->tableType = pTable->pMeta->pTableMeta->tableType; + return (SPhyNode*)node; +} + +static SPhyNode* createPseudoScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { + return initScanNode(pPlanNode, pTable, op, sizeof(SScanPhyNode)); } static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { - return initPhyNode(pPlanNode, OP_TagScan, sizeof(STagScanPhyNode)); + SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; + return createPseudoScanNode(pPlanNode, pTable, OP_TagScan); +} + +static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + // todo + return MASTER_SCAN; +} + +static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { + STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode)); + node->scanFlag = getScanFlag(pPlanNode, pTable); + node->window = pTable->window; + // todo tag cond + return (SPhyNode*)node; +} + +static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); +} + +static bool isSystemTable(SQueryTableInfo* pTable) { + // todo + return false; +} + +static bool needSeqScan(SQueryPlanNode* pPlanNode) { + // todo + return false; +} + +static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + if (isSystemTable(pTable)) { + return createPseudoScanNode(pPlanNode, pTable, OP_SystemTableScan); + } else if (needSeqScan(pPlanNode)) { + return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); + } + return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan); } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { @@ -58,53 +148,61 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL == pCxt->pCurrentSubplan->pChildern) { pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); } - taosArrayPush(pCxt->pCurrentSubplan->pChildern, subplan); + taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - taosArrayPush(subplan->pParents, pCxt->pCurrentSubplan); + taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } + SArray* currentLevel; + if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { + currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); + } else { + currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); + } + taosArrayPush(currentLevel, &subplan); pCxt->pCurrentSubplan = subplan; return subplan; } -static uint8_t getScanFlag(SQueryPlanNode* pPlanNode) { - // todo - return MASTER_SCAN; -} - -static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { - STableScanPhyNode* node = (STableScanPhyNode*)initPhyNode(pPlanNode, OP_TableScan, sizeof(STableScanPhyNode)); - node->scan.uid = pTable->pMeta->pTableMeta->uid; - node->scan.tableType = pTable->pMeta->pTableMeta->tableType; - node->scanFlag = getScanFlag(pPlanNode); - node->window = pTable->window; - // todo tag cond -} - static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { - // todo + epSet->inUse = 0; // todo + epSet->numOfEps = vg->numOfEps; + for (int8_t i = 0; i < vg->numOfEps; ++i) { + epSet->port[i] = vg->epAddr[i].port; + strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn); + } + return; } -static void splitSubplanBySTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { +static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList; for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { + STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); - subplan->pNode = createTableScanNode(pCxt, pPlanNode, pTable); - // todo reset pCxt->pCurrentSubplan + subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + RECOVERY_CURRENT_SUBPLAN(pCxt); } + return pCxt->nextId.templateId++; } -static SPhyNode* createExchangeNode() { - +static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) { + SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode)); + node->srcTemplateId = srcTemplateId; + return (SPhyNode*)node; } -static SPhyNode* createScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { +static bool needMultiNodeScan(SQueryTableInfo* pTable) { + // todo system table, for instance, user_tables + return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType); +} + +static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; - if (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType) { - splitSubplanBySTable(pCxt, pPlanNode, pTable); - return createExchangeNode(pCxt, pTable); + if (needMultiNodeScan(pTable)) { + return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } - return createTableScanNode(pCxt, pPlanNode, pTable); + return createSingleTableScanNode(pPlanNode, pTable); } static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { @@ -114,13 +212,13 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { node = createTagScanNode(pPlanNode); break; case QNODE_TABLESCAN: - node = createScanNode(pCxt, pPlanNode); + node = createTableScanNode(pCxt, pPlanNode); break; default: assert(false); } if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) { - node->pChildren = taosArrayInit(4, POINTER_BYTES); + node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); size_t size = taosArrayGetSize(pPlanNode->pChildren); for(int32_t i = 0; i < size; ++i) { SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); @@ -133,10 +231,8 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + ++(pCxt->nextId.templateId); subplan->pNode = createPhyNode(pCxt, pRoot); - SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - taosArrayPush(l0, &subplan); - taosArrayPush(pCxt->pDag->pSubplans, &l0); // todo deal subquery } @@ -144,7 +240,8 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD SPlanContext context = { .pCatalog = pCatalog, .pDag = calloc(1, sizeof(SQueryDag)), - .pCurrentSubplan = NULL + .pCurrentSubplan = NULL, + .nextId = {0} // todo queryid }; if (NULL == context.pDag) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -155,6 +252,6 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_SUCCESS; } -int32_t subPlanToString(struct SSubplan *pPhyNode, char** str) { - return TSDB_CODE_SUCCESS; +int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep) { + //todo } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c new file mode 100644 index 0000000000..2510797158 --- /dev/null +++ b/source/libs/planner/src/physicalPlanJson.c @@ -0,0 +1,779 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "plannerInt.h" +#include "parser.h" +#include "cJSON.h" + +typedef bool (*FToJson)(const void* obj, cJSON* json); +typedef bool (*FFromJson)(const cJSON* json, void* obj); + +static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) { + if (NULL == obj) { + return true; + } + + cJSON* jObj = cJSON_CreateObject(); + if (NULL == jObj || !func(obj, jObj)) { + cJSON_Delete(jObj); + return false; + } + return cJSON_AddItemToObject(json, name, jObj); +} + +static bool addItem(cJSON* json, FToJson func, const void* obj) { + cJSON* jObj = cJSON_CreateObject(); + if (NULL == jObj || !func(obj, jObj)) { + cJSON_Delete(jObj); + return false; + } + return cJSON_AddItemToArray(json, jObj); +} + +static bool fromObject(const cJSON* json, const char* name, FFromJson func, void* obj, bool required) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return !required; + } + return func(jObj, obj); +} + +static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, bool required) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return !required; + } + *obj = calloc(1, size); + if (NULL == *obj) { + return false; + } + return func(jObj, *obj); +} + +static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* array) { + size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); + if (size > 0) { + cJSON* jArray = cJSON_AddArrayToObject(json, name); + if (NULL == jArray) { + return false; + } + for (size_t i = 0; i < size; ++i) { + if (!addItem(jArray, func, taosArrayGetP(array, i))) { + return false; + } + } + } + return true; +} + +static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize) { + const cJSON* jArray = cJSON_GetObjectItem(json, name); + int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + if (size > 0) { + *array = taosArrayInit(size, POINTER_BYTES); + if (NULL == *array) { + return false; + } + } + for (int32_t i = 0; i < size; ++i) { + void* item = calloc(1, itemSize); + if (NULL == item || !func(cJSON_GetArrayItem(jArray, i), item)) { + return false; + } + taosArrayPush(*array, &item); + } + return true; +} + +static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) { + if (size > 0) { + cJSON* jArray = cJSON_AddArrayToObject(json, name); + if (NULL == jArray) { + return false; + } + for (size_t i = 0; i < size; ++i) { + if (!addItem(jArray, func, (const char*)array + itemSize * i)) { + return false; + } + } + } + return true; +} + +static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size) { + const cJSON* jArray = cJSON_GetObjectItem(json, name); + *size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + return jArray; +} + +static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) { + for (int32_t i = 0; i < size; ++i) { + if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) { + return false; + } + } + return true; +} + +static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { + const cJSON* jArray = getArray(json, name, size); + if (*size > 0) { + *array = calloc(1, itemSize * (*size)); + if (NULL == *array) { + return false; + } + } + return fromItem(jArray, func, *array, itemSize, *size); +} + +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { + const cJSON* jArray = getArray(json, name, size); + return fromItem(jArray, func, array, itemSize, *size); +} + +static char* getString(const cJSON* json, const char* name) { + char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); + char* res = calloc(1, strlen(p) + 1); + strcpy(res, p); + return res; +} + +static void copyString(const cJSON* json, const char* name, char* dst) { + strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); +} + +static int64_t getNumber(const cJSON* json, const char* name) { + return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); +} + +static const char* jkSchemaType = "Type"; +static const char* jkSchemaColId = "ColId"; +static const char* jkSchemaBytes = "Bytes"; +// The 'name' field do not need to be serialized. +static bool schemaToJson(const void* obj, cJSON* jSchema) { + const SSlotSchema* schema = (const SSlotSchema*)obj; + bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type); + if (res) { + res = cJSON_AddNumberToObject(jSchema, jkSchemaColId, schema->colId); + } + if (res) { + res = cJSON_AddNumberToObject(jSchema, jkSchemaBytes, schema->bytes); + } + return res; +} + +static bool schemaFromJson(const cJSON* json, void* obj) { + SSlotSchema* schema = (SSlotSchema*)obj; + schema->type = getNumber(json, jkSchemaType); + schema->colId = getNumber(json, jkSchemaColId); + schema->bytes = getNumber(json, jkSchemaBytes); + return true; +} + +static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; +static const char* jkColumnFilterInfoUpperRelOptr = "UpperRelOptr"; +static const char* jkColumnFilterInfoFilterstr = "Filterstr"; +static const char* jkColumnFilterInfoLowerBnd = "LowerBnd"; +static const char* jkColumnFilterInfoUpperBnd = "UpperBnd"; + +static bool columnFilterInfoToJson(const void* obj, cJSON* jFilter) { + const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj; + bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr); + if (res) { + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperRelOptr, filter->upperRelOptr); + } + if (res) { + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoFilterstr, filter->filterstr); + } + if (res) { + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerBnd, filter->lowerBndd); + } + if (res) { + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperBnd, filter->upperBndd); + } + return res; +} + +static bool columnFilterInfoFromJson(const cJSON* json, void* obj) { + SColumnFilterInfo* filter = (SColumnFilterInfo*)obj; + filter->lowerRelOptr = getNumber(json, jkColumnFilterInfoLowerRelOptr); + filter->upperRelOptr = getNumber(json, jkColumnFilterInfoUpperRelOptr); + filter->filterstr = getNumber(json, jkColumnFilterInfoFilterstr); + filter->lowerBndd = getNumber(json, jkColumnFilterInfoLowerBnd); + filter->upperBndd = getNumber(json, jkColumnFilterInfoUpperBnd); + return true; +} + +static const char* jkColumnInfoColId = "ColId"; +static const char* jkColumnInfoType = "Type"; +static const char* jkColumnInfoBytes = "Bytes"; +static const char* jkColumnInfoFilterList = "FilterList"; + +static bool columnInfoToJson(const void* obj, cJSON* jCol) { + const SColumnInfo* col = (const SColumnInfo*)obj; + bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId); + if (res) { + res = cJSON_AddNumberToObject(jCol, jkColumnInfoType, col->type); + } + if (res) { + res = cJSON_AddNumberToObject(jCol, jkColumnInfoBytes, col->bytes); + } + if (res) { + res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters); + } + return res; +} + +static bool columnInfoFromJson(const cJSON* json, void* obj) { + SColumnInfo* col = (SColumnInfo*)obj; + col->colId = getNumber(json, jkColumnInfoColId); + col->type = getNumber(json, jkColumnInfoType); + col->bytes = getNumber(json, jkColumnInfoBytes); + int32_t size = 0; + bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); + col->flist.numOfFilters = size; + return res; +} + +static const char* jkColumnTableId = "TableId"; +static const char* jkColumnFlag = "Flag"; +static const char* jkColumnInfo = "Info"; + +static bool columnToJson(const void* obj, cJSON* jCol) { + const SColumn* col = (const SColumn*)obj; + bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid); + if (res) { + res = cJSON_AddNumberToObject(jCol, jkColumnFlag, col->flag); + } + if (res) { + res = addObject(jCol, jkColumnInfo, columnInfoToJson, &col->info); + } + return res; +} + +static bool columnFromJson(const cJSON* json, void* obj) { + SColumn* col = (SColumn*)obj; + col->uid = getNumber(json, jkColumnTableId); + col->flag = getNumber(json, jkColumnFlag); + return fromObject(json, jkColumnInfo, columnInfoFromJson, &col->info, true); +} + +static bool exprNodeToJson(const void* obj, cJSON* jExprInfo); +static bool exprNodeFromJson(const cJSON* json, void* obj); + +static const char* jkExprNodeOper = "Oper"; +static const char* jkExprNodeLeft = "Left"; +static const char* jkExprNodeRight = "Right"; + +static bool operatorToJson(const void* obj, cJSON* jOper) { + const tExprNode* exprInfo = (const tExprNode*)obj; + bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr); + if (res) { + res = addObject(jOper, jkExprNodeLeft, exprNodeToJson, exprInfo->_node.pLeft); + } + if (res) { + res = addObject(jOper, jkExprNodeRight, exprNodeToJson, exprInfo->_node.pRight); + } + return res; +} + +static bool operatorFromJson(const cJSON* json, void* obj) { + tExprNode* exprInfo = (tExprNode*)obj; + exprInfo->_node.optr = getNumber(json, jkExprNodeOper); + bool res = fromObject(json, jkExprNodeLeft, exprNodeFromJson, exprInfo->_node.pLeft, false); + if (res) { + res = fromObject(json, jkExprNodeRight, exprNodeFromJson, exprInfo->_node.pRight, false); + } + return res; +} + +static const char* jkFunctionName = "Name"; +static const char* jkFunctionChild = "Child"; + +static bool functionToJson(const void* obj, cJSON* jFunc) { + const tExprNode* exprInfo = (const tExprNode*)obj; + bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName); + if (res) { + res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); + } + return res; +} + +static bool functionFromJson(const cJSON* json, void* obj) { + tExprNode* exprInfo = (tExprNode*)obj; + copyString(json, jkFunctionName, exprInfo->_function.functionName); + return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num); +} + +static const char* jkVariantType = "Type"; +static const char* jkVariantLen = "Len"; +static const char* jkVariantvalues = "values"; +static const char* jkVariantValue = "Value"; + +static bool variantToJson(const void* obj, cJSON* jVar) { + const SVariant* var = (const SVariant*)obj; + bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType); + if (res) { + res = cJSON_AddNumberToObject(jVar, jkVariantLen, var->nLen); + } + if (res) { + if (0/* in */) { + res = addArray(jVar, jkVariantvalues, variantToJson, var->arr); + } else if (IS_NUMERIC_TYPE(var->nType)) { + res = cJSON_AddNumberToObject(jVar, jkVariantValue, var->d); + } else { + res = cJSON_AddStringToObject(jVar, jkVariantValue, var->pz); + } + } + return res; +} + +static bool variantFromJson(const cJSON* json, void* obj) { + SVariant* var = (SVariant*)obj; + var->nType = getNumber(json, jkVariantType); + var->nLen = getNumber(json, jkVariantLen); + if (0/* in */) { + return fromArray(json, jkVariantvalues, variantFromJson, &var->arr, sizeof(SVariant)); + } else if (IS_NUMERIC_TYPE(var->nType)) { + var->d = getNumber(json, jkVariantValue); + } else { + var->pz = getString(json, jkVariantValue); + } + return true; +} + +static const char* jkExprNodeType = "Type"; +static const char* jkExprNodeOperator = "Operator"; +static const char* jkExprNodeFunction = "Function"; +static const char* jkExprNodeColumn = "Column"; +static const char* jkExprNodeValue = "Value"; + +static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) { + const tExprNode* exprInfo = (const tExprNode*)obj; + bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType); + if (res) { + switch (exprInfo->nodeType) { + case TEXPR_BINARYEXPR_NODE: + case TEXPR_UNARYEXPR_NODE: + res = addObject(jExprInfo, jkExprNodeOperator, operatorToJson, exprInfo); + break; + case TEXPR_FUNCTION_NODE: + res = addObject(jExprInfo, jkExprNodeFunction, functionToJson, exprInfo); + break; + case TEXPR_COL_NODE: + res = addObject(jExprInfo, jkExprNodeColumn, schemaToJson, exprInfo->pSchema); + break; + case TEXPR_VALUE_NODE: + res = addObject(jExprInfo, jkExprNodeValue, variantToJson, exprInfo->pVal); + break; + default: + res = false; + break; + } + } + return res; +} + +static bool exprNodeFromJson(const cJSON* json, void* obj) { + tExprNode* exprInfo = (tExprNode*)obj; + exprInfo->nodeType = getNumber(json, jkExprNodeType); + switch (exprInfo->nodeType) { + case TEXPR_BINARYEXPR_NODE: + case TEXPR_UNARYEXPR_NODE: + return fromObject(json, jkExprNodeOperator, operatorFromJson, exprInfo, false); + case TEXPR_FUNCTION_NODE: + return fromObject(json, jkExprNodeFunction, functionFromJson, exprInfo, false); + case TEXPR_COL_NODE: + return fromObject(json, jkExprNodeColumn, schemaFromJson, exprInfo->pSchema, false); + case TEXPR_VALUE_NODE: + return fromObject(json, jkExprNodeValue, variantFromJson, exprInfo->pVal, false); + default: + break; + } + return false; +} + +static const char* jkSqlExprSchema = "Schema"; +static const char* jkSqlExprColumns = "Columns"; +static const char* jkSqlExprInterBytes = "InterBytes"; +static const char* jkSqlExprParams = "Params"; +// token does not need to be serialized. +static bool sqlExprToJson(const void* obj, cJSON* jExpr) { + const SSqlExpr* expr = (const SSqlExpr*)obj; + bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema); + if (res) { + res = addRawArray(jExpr, jkSqlExprColumns, columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols); + } + if (res) { + res = cJSON_AddNumberToObject(jExpr, jkSqlExprInterBytes, expr->interBytes); + } + if (res) { + res = addRawArray(jExpr, jkSqlExprParams, variantToJson, expr->param, sizeof(SVariant), expr->numOfParams); + } + return res; +} + +static bool sqlExprFromJson(const cJSON* json, void* obj) { + SSqlExpr* expr = (SSqlExpr*)obj; + bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false); + if (res) { + res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), &expr->numOfCols); + } + if (res) { + expr->interBytes = getNumber(json, jkSqlExprInterBytes); + } + if (res) { + int32_t size = 0; + res = fromRawArray(json, jkSqlExprParams, variantFromJson, expr->param, sizeof(SVariant), &size); + expr->numOfParams = size; + } + return res; +} + +static const char* jkExprInfoBase = "Base"; +static const char* jkExprInfoExpr = "Expr"; + +static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) { + const SExprInfo* exprInfo = (const SExprInfo*)obj; + bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base); + if (res) { + res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, exprInfo->pExpr); + } + return res; +} + +static bool exprInfoFromJson(const cJSON* json, void* obj) { + SExprInfo* exprInfo = (SExprInfo*)obj; + bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true); + if (res) { + res = fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true); + } + return res; +} + +static const char* jkTimeWindowStartKey = "StartKey"; +static const char* jkTimeWindowEndKey = "EndKey"; + +static bool timeWindowToJson(const void* obj, cJSON* json) { + const STimeWindow* win = (const STimeWindow*)obj; + bool res = cJSON_AddNumberToObject(json, jkTimeWindowStartKey, win->skey); + if (res) { + res = cJSON_AddNumberToObject(json, jkTimeWindowEndKey, win->ekey); + } + return res; +} + +static bool timeWindowFromJson(const cJSON* json, void* obj) { + STimeWindow* win = (STimeWindow*)obj; + win->skey = getNumber(json, jkTimeWindowStartKey); + win->ekey = getNumber(json, jkTimeWindowEndKey); + return true; +} + +static const char* jkScanNodeTableId = "TableId"; +static const char* jkScanNodeTableType = "TableType"; + +static bool scanNodeToJson(const void* obj, cJSON* json) { + const SScanPhyNode* scan = (const SScanPhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); + } + return res; +} + +static bool scanNodeFromJson(const cJSON* json, void* obj) { + SScanPhyNode* scan = (SScanPhyNode*)obj; + scan->uid = getNumber(json, jkScanNodeTableId); + scan->tableType = getNumber(json, jkScanNodeTableType); + return true; +} + +static const char* jkTableScanNodeFlag = "Flag"; +static const char* jkTableScanNodeWindow = "Window"; +static const char* jkTableScanNodeTagsConditions = "TagsConditions"; + +static bool tableScanNodeToJson(const void* obj, cJSON* json) { + const STableScanPhyNode* scan = (const STableScanPhyNode*)obj; + bool res = scanNodeToJson(obj, json); + if (res) { + res = cJSON_AddNumberToObject(json, jkTableScanNodeFlag, scan->scanFlag); + } + if (res) { + res = addObject(json, jkTableScanNodeWindow, timeWindowToJson, &scan->window); + } + if (res) { + res = addArray(json, jkTableScanNodeTagsConditions, exprInfoToJson, scan->pTagsConditions); + } + return res; +} + +static bool tableScanNodeFromJson(const cJSON* json, void* obj) { + STableScanPhyNode* scan = (STableScanPhyNode*)obj; + bool res = scanNodeFromJson(json, obj); + if (res) { + scan->scanFlag = getNumber(json, jkTableScanNodeFlag); + } + if (res) { + res = fromObject(json, jkTableScanNodeWindow, timeWindowFromJson, &scan->window, true); + } + if (res) { + res = fromArray(json, jkTableScanNodeTagsConditions, exprInfoFromJson, &scan->pTagsConditions, sizeof(SExprInfo)); + } + return res; +} + +static const char* jkEpAddrFqdn = "Fqdn"; +static const char* jkEpAddrPort = "Port"; + +static bool epAddrToJson(const void* obj, cJSON* json) { + const SEpAddrMsg* ep = (const SEpAddrMsg*)obj; + bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); + if (res) { + res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port); + } + return res; +} + +static bool epAddrFromJson(const cJSON* json, void* obj) { + SEpAddrMsg* ep = (SEpAddrMsg*)obj; + copyString(json, jkEpAddrFqdn, ep->fqdn); + ep->port = getNumber(json, jkEpAddrPort); + return true; +} + +static const char* jkExchangeNodeSrcTemplateId = "SrcTemplateId"; +static const char* jkExchangeNodeSrcEndPoints = "SrcEndPoints"; + +static bool exchangeNodeToJson(const void* obj, cJSON* json) { + const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId); + if (res) { + res = addArray(json, jkExchangeNodeSrcEndPoints, epAddrToJson, exchange->pSrcEndPoints); + } + return res; +} + +static bool exchangeNodeFromJson(const cJSON* json, void* obj) { + SExchangePhyNode* exchange = (SExchangePhyNode*)obj; + exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId); + return fromArray(json, jkExchangeNodeSrcEndPoints, epAddrFromJson, &exchange->pSrcEndPoints, sizeof(SEpAddrMsg)); +} + +static bool specificPhyNodeToJson(const void* obj, cJSON* json) { + const SPhyNode* phyNode = (const SPhyNode*)obj; + switch (phyNode->info.type) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return tableScanNodeToJson(obj, json); + case OP_TagScan: + case OP_SystemTableScan: + return scanNodeToJson(obj, json); + case OP_Aggregate: + break; // todo + case OP_Project: + return true; + case OP_Groupby: + case OP_Limit: + case OP_SLimit: + case OP_TimeWindow: + case OP_SessionWindow: + case OP_StateWindow: + case OP_Fill: + case OP_MultiTableAggregate: + case OP_MultiTableTimeInterval: + case OP_Filter: + case OP_Distinct: + case OP_Join: + case OP_AllTimeWindow: + case OP_AllMultiTableTimeInterval: + case OP_Order: + break; // todo + case OP_Exchange: + return exchangeNodeToJson(obj, json); + default: + break; + } + return false; +} + +static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { + SPhyNode* phyNode = (SPhyNode*)obj; + switch (phyNode->info.type) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return tableScanNodeFromJson(json, obj); + case OP_TagScan: + case OP_SystemTableScan: + return scanNodeFromJson(json, obj); + case OP_Aggregate: + break; // todo + case OP_Project: + return true; + case OP_Groupby: + case OP_Limit: + case OP_SLimit: + case OP_TimeWindow: + case OP_SessionWindow: + case OP_StateWindow: + case OP_Fill: + case OP_MultiTableAggregate: + case OP_MultiTableTimeInterval: + case OP_Filter: + case OP_Distinct: + case OP_Join: + case OP_AllTimeWindow: + case OP_AllMultiTableTimeInterval: + case OP_Order: + break; // todo + case OP_Exchange: + return exchangeNodeFromJson(json, obj); + default: + break; + } + return false; +} + +static const char* jkPnodeName = "Name"; +static const char* jkPnodeTargets = "Targets"; +static const char* jkPnodeConditions = "Conditions"; +static const char* jkPnodeSchema = "Schema"; +static const char* jkPnodeChildren = "Children"; +// The 'pParent' field do not need to be serialized. +static bool phyNodeToJson(const void* obj, cJSON* jNode) { + const SPhyNode* phyNode = (const SPhyNode*)obj; + bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + if (res) { + res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets); + } + if (res) { + res = addArray(jNode, jkPnodeConditions, exprInfoToJson, phyNode->pConditions); + } + if (res) { + res = addRawArray(jNode, jkPnodeSchema, schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols); + } + if (res) { + res = addArray(jNode, jkPnodeChildren, phyNodeToJson, phyNode->pChildren); + } + if (res) { + res = addObject(jNode, phyNode->info.name, specificPhyNodeToJson, phyNode); + } + return res; +} + +static bool phyNodeFromJson(const cJSON* json, void* obj) { + SPhyNode* node = (SPhyNode*)obj; + node->info.name = getString(json, jkPnodeName); + node->info.type = opNameToOpType(node->info.name); + bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); + if (res) { + res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); + } + if (res) { + res = fromRawArray(json, jkPnodeSchema, schemaFromJson, node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); + } + if (res) { + res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); + } + if (res) { + res = fromObject(json, node->info.name, specificPhyNodeFromJson, node, true); + } + return res; +} + +static const char* jkIdQueryId = "QueryId"; +static const char* jkIdTemplateId = "TemplateId"; +static const char* jkIdSubplanId = "SubplanId"; + +static bool subplanIdToJson(const void* obj, cJSON* jId) { + const SSubplanId* id = (const SSubplanId*)obj; + bool res = cJSON_AddNumberToObject(jId, jkIdQueryId, id->queryId); + if (res) { + res = cJSON_AddNumberToObject(jId, jkIdTemplateId, id->templateId); + } + if (res) { + res = cJSON_AddNumberToObject(jId, jkIdSubplanId, id->subplanId); + } + return res; +} + +static bool subplanIdFromJson(const cJSON* json, void* obj) { + SSubplanId* id = (SSubplanId*)obj; + id->queryId = getNumber(json, jkIdQueryId); + id->templateId = getNumber(json, jkIdTemplateId); + id->subplanId = getNumber(json, jkIdSubplanId); + return true; +} + +static const char* jkSubplanId = "Id"; +static const char* jkSubplanNode = "Node"; + +static cJSON* subplanToJson(const SSubplan* subplan) { + cJSON* jSubplan = cJSON_CreateObject(); + if (NULL == jSubplan) { + return NULL; + } + + // The 'type', 'level', 'execEpSet', 'pChildern' and 'pParents' fields do not need to be serialized. + + bool res = addObject(jSubplan, jkSubplanId, subplanIdToJson, &subplan->id); + if (res) { + res = addObject(jSubplan, jkSubplanNode, phyNodeToJson, subplan->pNode); + } + + if (!res) { + cJSON_Delete(jSubplan); + return NULL; + } + return jSubplan; +} + +static SSubplan* subplanFromJson(const cJSON* json) { + SSubplan* subplan = calloc(1, sizeof(SSubplan)); + if (NULL == subplan) { + return NULL; + } + bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); + if (res) { + res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false); + } + + if (!res) { + qDestroySubplan(subplan); + return NULL; + } + return subplan; +} + +int32_t subPlanToString(const SSubplan* subplan, char** str) { + cJSON* json = subplanToJson(subplan); + if (NULL == json) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + *str = cJSON_Print(json); + return TSDB_CODE_SUCCESS; +} + +int32_t stringToSubplan(const char* str, SSubplan** subplan) { + cJSON* json = cJSON_Parse(str); + if (NULL == json) { + return TSDB_CODE_FAILED; + } + *subplan = subplanFromJson(json); + return (NULL == *subplan ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 19aac36e78..e9a4591d4a 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -13,606 +13,47 @@ * along with this program. If not, see . */ -#include "function.h" -#include "os.h" #include "parser.h" #include "plannerInt.h" -typedef struct SFillEssInfo { - int32_t fillType; // fill type - int64_t *val; // fill value -} SFillEssInfo; - -typedef struct SJoinCond { - bool tagExists; // denote if tag condition exists or not - SColumn *tagCond[2]; - SColumn *colCond[2]; -} SJoinCond; - -static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); -static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); - -int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len); -int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { - return 0; +void qDestroySubplan(SSubplan* pSubplan) { + // todo } -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); - assert(taosArrayGetSize(upstream) == 1); +void qDestroyQueryDag(struct SQueryDag* pDag) { + // todo +} - *pQueryNode = taosArrayGetP(upstream, 0); - - taosArrayDestroy(upstream); +int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) { + SQueryPlanNode* logicPlan; + int32_t code = createQueryPlan(pQueryInfo, &logicPlan); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + return code; + } + code = optimizeQueryPlan(logicPlan); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + return code; + } + code = createDag(logicPlan, NULL, pDag); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + qDestroyQueryDag(*pDag); + return code; + } + destroyQueryPlan(logicPlan); return TSDB_CODE_SUCCESS; } -int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { - return 0; +int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep) { + return setSubplanExecutionNode(subplan, templateId, ep); } -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { - - return 0; +int32_t qSubPlanToString(const SSubplan *subplan, char** str) { + return subPlanToString(subplan, str); } -int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str) { - return 0; -} - -void* destroyQueryPlan(SQueryPlanNode* pQueryNode) { - if (pQueryNode == NULL) { - return NULL; - } - - doDestroyQueryNode(pQueryNode); - return NULL; -} - -void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode) { - return NULL; -} - -//====================================================================================================================== - -static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { - SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); - - pNode->info.type = type; - pNode->info.name = strdup(name); - - pNode->numOfExpr = numOfOutput; - pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); - - for(int32_t i = 0; i < numOfOutput; ++i) { - taosArrayPush(pNode->pExpr, &pExpr[i]); - } - - pNode->pChildren = taosArrayInit(4, POINTER_BYTES); - for(int32_t i = 0; i < numOfPrev; ++i) { - taosArrayPush(pNode->pChildren, &prev[i]); - } - - switch(type) { - case QNODE_TAGSCAN: - case QNODE_TABLESCAN: { - SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); - memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); - info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); - pNode->pExtInfo = info; - break; - } - - case QNODE_TIMEWINDOW: { - SInterval* pInterval = calloc(1, sizeof(SInterval)); - pNode->pExtInfo = pInterval; - memcpy(pInterval, pExtInfo, sizeof(SInterval)); - break; - } - - case QNODE_STATEWINDOW: { - SColumn* psw = calloc(1, sizeof(SColumn)); - pNode->pExtInfo = psw; - memcpy(psw, pExtInfo, sizeof(SColumn)); - break; - } - - case QNODE_SESSIONWINDOW: { - SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); - pNode->pExtInfo = pSessionWindow; - memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); - break; - } - - case QNODE_GROUPBY: { - SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; - - SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - pGroupbyExpr->groupbyTag = p->groupbyTag; - pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); - - pNode->pExtInfo = pGroupbyExpr; - break; - } - - case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; - break; - } - - case QNODE_LIMIT: { - pNode->pExtInfo = calloc(1, sizeof(SLimit)); - memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); - break; - } - - case QNODE_SORT: { - pNode->pExtInfo = taosArrayDup(pExtInfo); - break; - } - - default: - break; - } - - return pNode; -} - -static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, - SArray* pExprs, SArray* tableCols) { - if (pQueryInfo->info.onlyTagQuery) { - int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info); - - if (pQueryInfo->info.distinct) { - pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); - } - - return pNode; - } - - SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); - - if (pQueryInfo->info.projectionQuery) { - int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); - } else { - STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); - - // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); - - SSourceParam param = {0}; - addIntoSourceParam(¶m, NULL, pCol); - SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); - pExpr[i] = p; - } - - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); - tfree(pExpr); - } - - return pNode; -} - -static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { - // group by column not by tag - size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); - - // check for aggregation - int32_t level = getExprFunctionLevel(pQueryInfo); - - for(int32_t i = level - 1; i >= 0; --i) { - SArray* p = pQueryInfo->exprList[i]; - size_t num = taosArrayGetSize(p); - - bool aggregateFunc = false; - for(int32_t j = 0; j < num; ++j) { - SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); - if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { - continue; - } - - aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); - if (aggregateFunc) { - break; - } - } - - if (aggregateFunc) { - if (pQueryInfo->interval.interval > 0) { - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); - } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); - } else if (pQueryInfo->stateWindow.col.info.colId > 0) { - pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); - } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); - } else { - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); - } - } else { - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); - } - } - - if (pQueryInfo->havingFieldNum > 0) { -// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); -// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); - pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); - - SArray* p = pQueryInfo->exprList[0]; // top expression in select clause - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); - } - - if (pQueryInfo->order != NULL) { - SArray* pList = pQueryInfo->exprList[0]; - pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); - } - - if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { - pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); - } - - return pNode; -} - -static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, - SArray* tableCols) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); - - SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; - - // handle the only tag query - SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); - if (pQueryInfo->info.onlyTagQuery) { - tfree(info.tableName); - return pNode; - } - - SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); - tfree(info.tableName); - return pNode1; -} - -static bool isAllAggExpr(SArray* pList) { - assert(pList != NULL); - - for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { - SExprInfo* p = taosArrayGetP(pList, k); - if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { - return false; - } - } - - return true; -} - -SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { - SArray* upstream = NULL; - - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause - upstream = taosArrayInit(4, POINTER_BYTES); - - size_t size = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < size; ++i) { - SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); - SArray* p = createQueryPlanImpl(pq); - taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); - } - } - - if (pQueryInfo->numOfTables > 1) { // it is a join query - // 1. separate the select clause according to table - taosArrayDestroy(upstream); - upstream = taosArrayInit(5, POINTER_BYTES); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; - uint64_t uid = pTableMetaInfo->pTableMeta->uid; - - SArray* exprList = taosArrayInit(4, POINTER_BYTES); - if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; -// dropAllExprInfo(exprList); - exit(-1); - } - - // 2. create the query execution node - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; - - // 3. get the required table column list - SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); - columnListCopy(tableColumnList, pQueryInfo->colList, uid); - - // 4. add the projection query node - SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); - columnListDestroy(tableColumnList); -// dropAllExprInfo(exprList); - taosArrayPush(upstream, &pNode); - } - - // 3. add the join node here - SQueryTableInfo info = {0}; - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); - SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList[0]->pData, num, NULL); - - // 4. add the aggregation or projection execution node - pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } else { // only one table, normal query process - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } - - return upstream; -} - -static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - tfree(pQueryNode->pExtInfo); - tfree(pQueryNode->pSchema); - tfree(pQueryNode->info.name); -// dropAllExprInfo(pQueryNode->pExpr); - - if (pQueryNode->pChildren != NULL) { - int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); - for(int32_t i = 0; i < size; ++i) { - SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i); - doDestroyQueryNode(p); - } - - taosArrayDestroy(pQueryNode->pChildren); - } - - tfree(pQueryNode); -} - -static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { - if (level > 0) { - sprintf(buf + totalLen, "%*c", level, ' '); - totalLen += level; - } - - int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); - int32_t len = len1 + totalLen; - - switch(pQueryNode->info.type) { - case QNODE_TABLESCAN: { - SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, - pInfo->window.skey, pInfo->window.ekey); - assert(len1 > 0); - len += len1; - - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); - len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); - - assert(len1 > 0); - len += len1; - } - - len1 = sprintf(buf + len, "\n"); - assert(len1 > 0); - - len += len1; - break; - } - - case QNODE_PROJECT: { - len1 = sprintf(buf + len, "cols:"); - assert(len1 > 0); - len += len1; - - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ")"); - len += len1; - - // todo print filter info - len1 = sprintf(buf + len, " filters:(nil)\n"); - len += len1; - break; - } - - case QNODE_AGGREGATE: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ")\n"); - len += len1; - - break; - } - - case QNODE_TIMEWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ") "); - len += len1; - - SInterval* pInterval = pQueryNode->pExtInfo; - - // todo dynamic return the time precision - len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", - pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, - TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); - len += len1; - - break; - } - - case QNODE_STATEWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ") "); - len += len1; - - SColumn* pCol = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); - len += len1; - break; - } - - case QNODE_SESSIONWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - - len1 = sprintf(buf + len, ") "); - len += len1; - - struct SSessionWindow* ps = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); - len += len1; - break; - } - - case QNODE_GROUPBY: { - len = printExprInfo(buf, pQueryNode, len); - - SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, ") groupby_col: "); - len += len1; - - for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { - SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); - len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); - len += len1; - } - - len += sprintf(buf + len, "\n"); - break; - } - - case QNODE_FILL: { - SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%d", pEssInfo->fillType); - len += len1; - - if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { - len1 = sprintf(buf + len, ", val:"); - len += len1; - - // todo get the correct fill data type - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); - len += len1; - - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_LIMIT: { - SLimit* pVal = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); - len += len1; - break; - } - - case QNODE_DISTINCT: - case QNODE_TAGSCAN: { - len1 = sprintf(buf + len, "cols: "); - len += len1; - - len = printExprInfo(buf, pQueryNode, len); - - len1 = sprintf(buf + len, ")\n"); - len += len1; - - break; - } - - case QNODE_SORT: { - len1 = sprintf(buf + len, "cols:"); - len += len1; - - SArray* pSort = pQueryNode->pExtInfo; - for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { - SOrder* p = taosArrayGet(pSort, i); - len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); - - len += len1; - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_JOIN: { - // print join condition - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - } - - return len; -} - -int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { - int32_t len1 = 0; - - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* pExpr = &pExprInfo->base; - len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - assert(len1 > 0); - - len += len1; - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - return len; -} - -int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { - int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); - - for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) { - SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i); - int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); - len = len1; - } - - return len; -} - -int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { - assert(pQueryNode); - - *str = calloc(1, 4096); - - int32_t len = sprintf(*str, "===== logic plan =====\n"); - queryPlanToStringImpl(*str, pQueryNode, 0, len); - - return TSDB_CODE_SUCCESS; -} - -SQueryPlanNode* queryPlanFromString() { - return NULL; +int32_t qStringToSubplan(const char* str, SSubplan** subplan) { + return stringToSubplan(str, subplan); } diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index f00adfaeb2..7fbfcfe7ef 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -5,14 +5,20 @@ MESSAGE(STATUS "build planner unit test") SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) -ADD_EXECUTABLE(plannerTest ${SOURCE_LIST}) +ADD_EXECUTABLE(plannerTest + ${SOURCE_LIST} + "${SOURCE_LIST}/../../../parser/test/mockCatalog.cpp" + "${SOURCE_LIST}/../../../parser/test/mockCatalogService.cpp" +) + TARGET_LINK_LIBRARIES( plannerTest - PUBLIC os util common planner parser catalog transport gtest function query + PUBLIC os util common planner parser catalog transport gtest function qcom ) TARGET_INCLUDE_DIRECTORIES( plannerTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/planner/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/planner/inc" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/test" ) diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp new file mode 100644 index 0000000000..3be3337304 --- /dev/null +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "plannerInt.h" +#include "mockCatalogService.h" + +using namespace std; +using namespace testing; + +void* myCalloc(size_t nmemb, size_t size) { + if (void* p = calloc(nmemb, size)) { + return p; + } + throw bad_alloc(); +} + +class PhyPlanTest : public Test { +protected: + void pushScan(const string& db, const string& table, int32_t scanOp) { + shared_ptr meta = mockCatalogService->getTableMeta(db, table); + EXPECT_TRUE(meta); +// typedef struct SQueryPlanNode { +// SArray *pExpr; // the query functions or sql aggregations +// int32_t numOfExpr; // number of result columns, which is also the number of pExprs +// } SQueryPlanNode; + unique_ptr scan((SQueryPlanNode*)calloc(1, sizeof(SQueryPlanNode))); + scan->info.type = scanOp; + scan->numOfCols = meta->schema->tableInfo.numOfColumns; + scan->pSchema = (SSchema*)myCalloc(1, sizeof(SSchema) * scan->numOfCols); + memcpy(scan->pSchema, meta->schema->schema, sizeof(SSchema) * scan->numOfCols); + //todo 'pExpr' 'numOfExpr' + scan->pExtInfo = createScanExtInfo(meta); + pushNode(scan.release()); + } + + int32_t run() { + SQueryDag* dag = nullptr; + int32_t code = createDag(logicPlan_.get(), nullptr, &dag); + dag_.reset(dag); + return code; + } + + void explain() { + size_t level = taosArrayGetSize(dag_->pSubplans); + for (size_t i = 0; i < level; ++i) { + std::cout << "level " << i << ":" << std::endl; + const SArray* subplans = (const SArray*)taosArrayGetP(dag_->pSubplans, i); + size_t num = taosArrayGetSize(subplans); + for (size_t j = 0; j < num; ++j) { + std::cout << "no " << j << ":" << std::endl; + char* str = nullptr; + ASSERT_EQ (TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str)); + std::cout << str << std::endl; + free(str); + } + } + } + + SQueryDag* reslut() { + return dag_.get(); + } + +private: + void pushNode(SQueryPlanNode* node) { + if (logicPlan_) { + // todo + } else { + logicPlan_.reset(node); + } + } + + void copySchemaMeta(STableMeta** dst, const STableMeta* src) { + int32_t size = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); + *dst = (STableMeta*)myCalloc(1, size); + memcpy(*dst, src, size); + } + + void copyStorageMeta(SVgroupsInfo** dst, const std::vector& src) { + *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * src.size()); + (*dst)->numOfVgroups = src.size(); + for (int32_t i = 0; i < src.size(); ++i) { + (*dst)->vgroups[i].vgId = src[i].vgId; + (*dst)->vgroups[i].numOfEps = src[i].numOfEps; + memcpy((*dst)->vgroups[i].epAddr, src[i].epAddr, src[i].numOfEps); + } + } + + SQueryTableInfo* createScanExtInfo(shared_ptr& meta) { + SQueryTableInfo* info = (SQueryTableInfo*)myCalloc(1, sizeof(SQueryTableInfo)); + info->pMeta = (STableMetaInfo*)myCalloc(1, sizeof(STableMetaInfo)); + copySchemaMeta(&info->pMeta->pTableMeta, meta->schema.get()); + copyStorageMeta(&info->pMeta->vgroupList, meta->vgs); + return info; + } + + shared_ptr meta_; + unique_ptr logicPlan_; + unique_ptr dag_; +}; + +// select * from table +TEST_F(PhyPlanTest, tableScanTest) { + pushScan("root.test", "t1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = reslut(); + // todo check +} + +// select * from supertable +TEST_F(PhyPlanTest, superTableScanTest) { + pushScan("root.test", "st1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = reslut(); + // todo check +} diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 5ede8dd155..9379b06ac8 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -20,6 +20,7 @@ #include "taos.h" #include "parser.h" +#include "mockCatalog.h" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -27,9 +28,25 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); +class ParserEnv : public testing::Environment { +public: + virtual void SetUp() { + initMetaDataEnv(); + generateMetaData(); + } + + virtual void TearDown() { + destroyMetaDataEnv(); + } + + ParserEnv() {} + virtual ~ParserEnv() {} +}; + +int main(int argc, char* argv[]) { + testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } TEST(testCase, planner_test) { diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt new file mode 100644 index 0000000000..41cf1826bc --- /dev/null +++ b/source/libs/qcom/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QUERY_SRC) +add_library(qcom ${QUERY_SRC}) +target_include_directories( + qcom + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + qcom + PRIVATE os util transport +) diff --git a/source/libs/qcom/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h new file mode 100644 index 0000000000..75c1e134cd --- /dev/null +++ b/source/libs/qcom/inc/queryInt.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QUERY_INT_H_ +#define _TD_QUERY_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c new file mode 100644 index 0000000000..2a13b708ec --- /dev/null +++ b/source/libs/qcom/src/queryUtil.c @@ -0,0 +1,78 @@ +#include "os.h" +#include "taosmsg.h" + +#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) +#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) + +static struct SSchema _s = { + .colId = TSDB_TBNAME_COLUMN_INDEX, + .type = TSDB_DATA_TYPE_BINARY, + .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, + .name = "tbname", +}; + +SSchema* tGetTbnameColumnSchema() { + return &_s; +} + +static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { + int32_t rowLen = 0; + + for (int32_t i = 0; i < numOfCols; ++i) { + // 1. valid types + if (!isValidDataType(pSchema[i].type)) { + return false; + } + + // 2. valid length for each type + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { + if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { + return false; + } + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { + return false; + } + } else { + if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { + return false; + } + } + + // 3. valid column names + for (int32_t j = i + 1; j < numOfCols; ++j) { + if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { + return false; + } + } + + rowLen += pSchema[i].bytes; + } + + return rowLen <= maxLen; +} + +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { + if (!VALIDNUMOFCOLS(numOfCols)) { + return false; + } + + if (!VALIDNUMOFTAGS(numOfTags)) { + return false; + } + + /* first column must be the timestamp, which is a primary key */ + if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { + return false; + } + + if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { + return false; + } + + if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { + return false; + } + + return true; +} \ No newline at end of file diff --git a/source/libs/query/src/querymsg.c b/source/libs/qcom/src/querymsg.c similarity index 97% rename from source/libs/query/src/querymsg.c rename to source/libs/qcom/src/querymsg.c index 9d99b568a5..9ca8b6789b 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -39,7 +39,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 STableInfoMsg *bMsg = (STableInfoMsg *)*msg; - bMsg->msgHead.vgId = bInput->vgId; + bMsg->vgId = bInput->vgId; strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; @@ -120,7 +120,7 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pRsp->vgroupInfo[i].hashEnd = htonl(pRsp->vgroupInfo[i].hashEnd); for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); + pRsp->vgroupInfo[i].epAddr[n].port = htons(pRsp->vgroupInfo[i].epAddr[n].port); } if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { @@ -305,7 +305,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -323,7 +323,7 @@ void msgInit() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; diff --git a/source/libs/query/CMakeLists.txt b/source/libs/query/CMakeLists.txt deleted file mode 100644 index 579a4b279c..0000000000 --- a/source/libs/query/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -aux_source_directory(src QUERY_SRC) -add_library(query ${QUERY_SRC}) -target_include_directories( - query - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/query" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - query - PRIVATE os util common transport -) diff --git a/source/libs/query/inc/queryInt.h b/source/libs/query/inc/queryInt.h deleted file mode 100644 index f3204b3785..0000000000 --- a/source/libs/query/inc/queryInt.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_QUERY_INT_H_ -#define _TD_QUERY_INT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - - -#include "tlog.h" - -extern int32_t qDebugFlag; - -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_QUERY_INT_H_*/ diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index fd00085381..6baaab1ef4 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,5 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner common -) \ No newline at end of file + PRIVATE os util planner qcom common catalog transport +) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index b1b128e200..73e7c4d24e 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -24,15 +24,86 @@ extern "C" { #include "tarray.h" #include "planner.h" #include "scheduler.h" +#include "thash.h" -typedef struct SQuery { - SArray **pSubquery; - int32_t numOfLevels; - int32_t currentLevel; -} SQuery; +#define SCHEDULE_DEFAULT_JOB_NUMBER 1000 +#define SCHEDULE_DEFAULT_TASK_NUMBER 1000 + +#define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA + +enum { + SCH_STATUS_NOT_START = 1, + SCH_STATUS_EXECUTING, + SCH_STATUS_SUCCEED, + SCH_STATUS_FAILED, + SCH_STATUS_CANCELLING, + SCH_STATUS_CANCELLED +}; + +typedef struct SSchedulerMgmt { + uint64_t taskId; + SSchedulerCfg cfg; + SHashObj *Jobs; // key: queryId, value: SQueryJob* +} SSchedulerMgmt; + +typedef struct SQueryTask { + uint64_t taskId; // task id + SSubplan *plan; // subplan + char *msg; // operator tree + int8_t status; // task status + SEpAddr execAddr; // task actual executed node address + SQueryProfileSummary summary; // task execution summary + int32_t childReady; // child task ready number + SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* + SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* +} SQueryTask; + +typedef struct SQueryLevel { + int32_t level; + int8_t status; + int32_t taskNum; + SArray *subTasks; // Element is SQueryTask +} SQueryLevel; + +typedef struct SQueryJob { + uint64_t queryId; + int32_t levelNum; + int32_t levelIdx; + int8_t status; + SQueryProfileSummary summary; + SEpSet dataSrcEps; + SEpAddr resEp; + void *transport; + SArray *qnodeList; + tsem_t rspSem; + int32_t userFetch; + int32_t remoteFetch; + void *res; + + SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* + SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* + + SArray *levels; // Element is SQueryLevel, starting from 0. + SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. +} SQueryJob; + +#define SCH_HAS_QNODE_IN_CLUSTER(type) (false) //TODO CLUSTER TYPE +#define SCH_TASK_READY_TO_LUNCH(task) ((task)->childReady >= taosArrayGetSize((task)->children)) // MAY NEED TO ENHANCE +#define SCH_IS_DATA_SRC_TASK(task) (task->plan->type == QUERY_TYPE_SCAN) + +#define SCH_JOB_ERR_LOG(param, ...) qError("QID:%"PRIx64 param, job->queryId, __VA_ARGS__) +#define SCH_TASK_ERR_LOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) + +#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define SCH_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) +#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + +extern int32_t schLaunchTask(SQueryJob *job, SQueryTask *task); #ifdef __cplusplus } #endif -#endif /*_TD_SCHEDULER_INT_H_*/ \ No newline at end of file +#endif /*_TD_SCHEDULER_INT_H_*/ diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 37f6240f9b..6014ff9ab6 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -13,4 +13,674 @@ * along with this program. If not, see . */ -#include "schedulerInt.h" \ No newline at end of file +#include "schedulerInt.h" +#include "taosmsg.h" +#include "query.h" +#include "catalog.h" + +SSchedulerMgmt schMgmt = {0}; + + +int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_fn_t fp) { +/* + SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT); + if (pRequest == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SRequestMsgBody body = {0}; + buildConnectMsg(pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + + if (pRequest->code != TSDB_CODE_SUCCESS) { + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + printf("failed to connect to server, reason: %s\n\n", errorMsg); + + destroyRequest(pRequest); + taos_close(pTscObj); + pTscObj = NULL; + } else { + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); + destroyRequest(pRequest); + } +*/ +} + +int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { + for (int32_t i = 0; i < job->levelNum; ++i) { + SQueryLevel *level = taosArrayGet(job->levels, i); + + for (int32_t m = 0; m < level->taskNum; ++m) { + SQueryTask *task = taosArrayGet(level->subTasks, m); + SSubplan *plan = task->plan; + int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0; + int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0; + + if (childNum > 0) { + task->children = taosArrayInit(childNum, POINTER_BYTES); + if (NULL == task->children) { + qError("taosArrayInit %d failed", childNum); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + for (int32_t n = 0; n < childNum; ++n) { + SSubplan *child = taosArrayGet(plan->pChildern, n); + SQueryTask *childTask = taosHashGet(planToTask, &child, POINTER_BYTES); + if (childTask) { + qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + if (NULL == taosArrayPush(task->children, &childTask)) { + qError("taosArrayPush failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (parentNum > 0) { + task->parents = taosArrayInit(parentNum, POINTER_BYTES); + if (NULL == task->parents) { + qError("taosArrayInit %d failed", parentNum); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + for (int32_t n = 0; n < parentNum; ++n) { + SSubplan *parent = taosArrayGet(plan->pParents, n); + SQueryTask *parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); + if (parentTask) { + qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + if (NULL == taosArrayPush(task->parents, &parentTask)) { + qError("taosArrayPush failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } + } + + SQueryLevel *level = taosArrayGet(job->levels, 0); + if (level->taskNum > 1) { + qError("invalid plan info, level 0, taskNum:%d", level->taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + SQueryTask *task = taosArrayGet(level->subTasks, 0); + if (task->parents && taosArrayGetSize(task->parents) > 0) { + qError("invalid plan info, level 0, parentNum:%d", (int32_t)taosArrayGetSize(task->parents)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + + return TSDB_CODE_SUCCESS; +} + + +int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { + int32_t code = 0; + + job->queryId = dag->queryId; + + if (dag->numOfSubplans <= 0) { + qError("invalid subplan num:%d", dag->numOfSubplans); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t levelNum = (int32_t)taosArrayGetSize(dag->pSubplans); + if (levelNum <= 0) { + qError("invalid level num:%d", levelNum); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SHashObj *planToTask = taosHashInit(SCHEDULE_DEFAULT_TASK_NUMBER, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == planToTask) { + qError("taosHashInit %d failed", SCHEDULE_DEFAULT_TASK_NUMBER); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + job->levels = taosArrayInit(levelNum, sizeof(SQueryLevel)); + if (NULL == job->levels) { + qError("taosArrayInit %d failed", levelNum); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + job->levelNum = levelNum; + job->levelIdx = levelNum - 1; + + job->subPlans = dag->pSubplans; + + SQueryLevel level = {0}; + SArray *levelPlans = NULL; + int32_t levelPlanNum = 0; + + level.status = SCH_STATUS_NOT_START; + + for (int32_t i = 0; i < levelNum; ++i) { + level.level = i; + levelPlans = taosArrayGetP(dag->pSubplans, i); + if (NULL == levelPlans) { + qError("no level plans for level %d", i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + levelPlanNum = (int32_t)taosArrayGetSize(levelPlans); + if (levelPlanNum <= 0) { + qError("invalid level plans number:%d, level:%d", levelPlanNum, i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + level.taskNum = levelPlanNum; + + level.subTasks = taosArrayInit(levelPlanNum, sizeof(SQueryTask)); + if (NULL == level.subTasks) { + qError("taosArrayInit %d failed", levelPlanNum); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + for (int32_t n = 0; n < levelPlanNum; ++n) { + SSubplan *plan = taosArrayGet(levelPlans, n); + SQueryTask task = {0}; + + task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + task.plan = plan; + task.status = SCH_STATUS_NOT_START; + + void *p = taosArrayPush(level.subTasks, &task); + if (NULL == p) { + qError("taosArrayPush failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { + qError("taosHashPut failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (NULL == taosArrayPush(job->levels, &level)) { + qError("taosArrayPush failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + SCH_ERR_JRET(schBuildTaskRalation(job, planToTask)); + + if (planToTask) { + taosHashCleanup(planToTask); + } + + return TSDB_CODE_SUCCESS; + +_return: + if (level.subTasks) { + taosArrayDestroy(level.subTasks); + } + + if (planToTask) { + taosHashCleanup(planToTask); + } + + SCH_RET(code); +} + +int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) { + if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) { + return TSDB_CODE_SUCCESS; + } + + int32_t qnodeNum = taosArrayGetSize(job->qnodeList); + + for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) { + SEpAddr *addr = taosArrayGet(job->qnodeList, i); + + strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn)); + epSet->port[epSet->numOfEps] = addr->port; + + ++epSet->numOfEps; + } + + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && epSet->numOfEps < tListLen(epSet->port); ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t schPushTaskToExecList(SQueryJob *job, SQueryTask *task) { + if (0 != taosHashPut(job->execTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { + qError("taosHashPut failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schMoveTaskToSuccList(SQueryJob *job, SQueryTask *task, bool *moved) { + if (0 != taosHashRemove(job->execTasks, &task->taskId, sizeof(task->taskId))) { + qWarn("remove task[%"PRIx64"] from execTasks failed", task->taskId); + return TSDB_CODE_SUCCESS; + } + + if (0 != taosHashPut(job->execTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { + qError("taosHashPut failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + *moved = true; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { + int32_t msgSize = 0; + void *msg = NULL; + + switch (msgType) { + case TSDB_MSG_TYPE_QUERY: { + if (NULL == task->msg) { + qError("query msg is NULL"); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + int32_t len = strlen(task->msg); + msgSize = sizeof(SSchedulerQueryMsg) + len; + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchedulerQueryMsg *pMsg = msg; + pMsg->queryId = job->queryId; + pMsg->taskId = task->taskId; + pMsg->contentLen = len; + memcpy(pMsg->msg, task->msg, len); + break; + } + case TSDB_MSG_TYPE_RSP_READY: { + msgSize = sizeof(SSchedulerReadyMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchedulerReadyMsg *pMsg = msg; + pMsg->queryId = job->queryId; + pMsg->taskId = task->taskId; + break; + } + case TSDB_MSG_TYPE_FETCH: { + msgSize = sizeof(SSchedulerFetchMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchedulerFetchMsg *pMsg = msg; + pMsg->queryId = job->queryId; + pMsg->taskId = task->taskId; + break; + } + default: + qError("unknown msg type:%d", msgType); + break; + } + + //TODO SEND MSG + + return TSDB_CODE_SUCCESS; +} + +int32_t schTaskCheckAndSetRetry(SQueryJob *job, SQueryTask *task, int32_t errCode, bool *needRetry) { + // TODO set retry or not based on task type/errCode/retry times/job status/available eps... + // TODO if needRetry, set task retry info + + *needRetry = false; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schFetchFromRemote(SQueryJob *job) { + int32_t code = 0; + + if (atomic_val_compare_exchange_32(&job->remoteFetch, 0, 1) != 0) { + qInfo("prior fetching not finished"); + return TSDB_CODE_SUCCESS; + } + + SCH_ERR_JRET(schAsyncSendMsg(job, NULL, TSDB_MSG_TYPE_FETCH)); + + return TSDB_CODE_SUCCESS; + +_return: + atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + return code; +} + + +int32_t schProcessOnJobSuccess(SQueryJob *job) { + job->status = SCH_STATUS_SUCCEED; + + if (job->userFetch) { + SCH_ERR_RET(schFetchFromRemote(job)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnJobFailure(SQueryJob *job) { + job->status = SCH_STATUS_FAILED; + + atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + if (job->userFetch) { + tsem_post(&job->rspSem); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnDataFetched(SQueryJob *job) { + atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + tsem_post(&job->rspSem); +} + + +int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { + bool moved = false; + + SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved)); + if (!moved) { + SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); + return TSDB_CODE_SUCCESS; + } + + task->status = SCH_STATUS_SUCCEED; + + int32_t parentNum = (int32_t)taosArrayGetSize(task->parents); + if (parentNum == 0) { + if (task->plan->level != 0) { + qError("level error"); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + strncpy(job->resEp.fqdn, task->execAddr.fqdn, sizeof(job->resEp.fqdn)); + job->resEp.port = task->execAddr.port; + + SCH_ERR_RET(schProcessOnJobSuccess(job)); + + return TSDB_CODE_SUCCESS; + } + + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CONDIDATE_EP_NUM) { + strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); + job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; + + ++job->dataSrcEps.numOfEps; + } + + for (int32_t i = 0; i < parentNum; ++i) { + SQueryTask *par = taosArrayGet(task->parents, i); + + ++par->childReady; + + SCH_ERR_RET(qSetSubplanExecutionNode(par->plan, task->plan->id.templateId, &task->execAddr)); + + if (SCH_TASK_READY_TO_LUNCH(par)) { + SCH_ERR_RET(schLaunchTask(job, task)); + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCode) { + bool needRetry = false; + SCH_ERR_RET(schTaskCheckAndSetRetry(job, task, errCode, &needRetry)); + + if (!needRetry) { + SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); + + job->status = SCH_STATUS_FAILED; + SCH_ERR_RET(schProcessOnJobFailure(job)); + + return TSDB_CODE_SUCCESS; + } + + SCH_ERR_RET(schLaunchTask(job, task)); + + return TSDB_CODE_SUCCESS; +} + +int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32_t rspCode) { + int32_t code = 0; + + switch (msgType) { + case TSDB_MSG_TYPE_QUERY: + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); + } else { + code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RSP_READY); + if (code) { + goto _task_error; + } + } + break; + case TSDB_MSG_TYPE_RSP_READY: + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); + } else { + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; + } + } + break; + case TSDB_MSG_TYPE_FETCH: + SCH_ERR_JRET(rspCode); + SCH_ERR_JRET(schProcessOnDataFetched(job)); + break; + default: + qError("unknown msg type:%d received", msgType); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + return TSDB_CODE_SUCCESS; + +_task_error: + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, code)); + return TSDB_CODE_SUCCESS; + +_return: + code = schProcessOnJobFailure(job); + return code; +} + + + + +int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { + SSubplan *plan = task->plan; + + SCH_ERR_RET(qSubPlanToString(plan, &task->msg)); + if (plan->execEpSet.numOfEps <= 0) { + SCH_ERR_RET(schSetTaskExecEpSet(job, &plan->execEpSet)); + } + + if (plan->execEpSet.numOfEps <= 0) { + SCH_TASK_ERR_LOG("invalid execEpSet num:%d", plan->execEpSet.numOfEps); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + SCH_ERR_RET(schAsyncSendMsg(job, task, TSDB_MSG_TYPE_QUERY)); + + SCH_ERR_RET(schPushTaskToExecList(job, task)); + + task->status = SCH_STATUS_EXECUTING; + + return TSDB_CODE_SUCCESS; +} + +int32_t schLaunchJob(SQueryJob *job) { + SQueryLevel *level = taosArrayGet(job->levels, job->levelIdx); + for (int32_t i = 0; i < level->taskNum; ++i) { + SQueryTask *task = taosArrayGet(level->subTasks, i); + SCH_ERR_RET(schLaunchTask(job, task)); + } + + job->status = SCH_STATUS_EXECUTING; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schedulerInit(SSchedulerCfg *cfg) { + schMgmt.Jobs = taosHashInit(SCHEDULE_DEFAULT_JOB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == schMgmt.Jobs) { + SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER); + } + + if (cfg) { + schMgmt.cfg = *cfg; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { + if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (taosArrayGetSize(qnodeList) <= 0) { + qInfo("qnodeList is empty"); + } + + int32_t code = 0; + SQueryJob *job = calloc(1, sizeof(SQueryJob)); + if (NULL == job) { + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + job->transport = transport; + job->qnodeList = qnodeList; + + SCH_ERR_JRET(schValidateAndBuildJob(pDag, job)); + + job->execTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == job->execTasks) { + qError("taosHashInit %d failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + job->succTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == job->succTasks) { + qError("taosHashInit %d failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + tsem_init(&job->rspSem, 0, 0); + + if (0 != taosHashPut(schMgmt.Jobs, &job->queryId, sizeof(job->queryId), &job, POINTER_BYTES)) { + qError("taosHashPut queryId:%"PRIx64" failed", job->queryId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + job->status = SCH_STATUS_NOT_START; + + SCH_ERR_JRET(schLaunchJob(job)); + + *(SQueryJob **)pJob = job; + + return TSDB_CODE_SUCCESS; + +_return: + + *(SQueryJob **)pJob = NULL; + scheduleFreeJob(job); + + SCH_RET(code); +} + +int32_t scheduleFetchRows(void *pJob, void **data) { + if (NULL == pJob || NULL == data) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SQueryJob *job = pJob; + int32_t code = 0; + + if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { + qError("prior fetching not finished"); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (job->status == SCH_STATUS_SUCCEED) { + SCH_ERR_JRET(schFetchFromRemote(job)); + } + + tsem_wait(&job->rspSem); + + *data = job->res; + job->res = NULL; + +_return: + atomic_val_compare_exchange_32(&job->userFetch, 1, 0); + + return code; +} + +int32_t scheduleCancelJob(void *pJob) { + //TODO + + return TSDB_CODE_SUCCESS; +} + +void scheduleFreeJob(void *pJob) { + if (NULL == pJob) { + return; + } + + SQueryJob *job = pJob; + + if (job->status > 0) { + if (0 != taosHashRemove(schMgmt.Jobs, &job->queryId, sizeof(job->queryId))) { + qError("remove job:%"PRIx64"from mgmt failed", job->queryId); // maybe already freed + return; + } + + if (job->status == SCH_STATUS_EXECUTING) { + scheduleCancelJob(pJob); + } + } + + //TODO free job +} + +void schedulerDestroy(void) { + if (schMgmt.Jobs) { + taosHashCleanup(schMgmt.Jobs); //TODO + schMgmt.Jobs = NULL; + } +} + + diff --git a/source/libs/scheduler/test/CMakeLists.txt b/source/libs/scheduler/test/CMakeLists.txt new file mode 100644 index 0000000000..00a6d08e5d --- /dev/null +++ b/source/libs/scheduler/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scheduler unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos planner scheduler +) + +TARGET_INCLUDE_DIRECTORIES( + schedulerTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scheduler/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scheduler/inc" +) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index e69de29bb2..9e94553058 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "scheduler.h" +#include "tep.h" +#include "trpc.h" + +namespace { +void mockBuildDag(SQueryDag *dag) { + uint64_t qId = 0x111111111111; + + dag->queryId = qId; + dag->numOfSubplans = 2; + dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); + SArray *scan = taosArrayInit(1, sizeof(SSubplan)); + SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + + SSubplan scanPlan = {0}; + SSubplan mergePlan = {0}; + + scanPlan.id.queryId = qId; + scanPlan.id.templateId = 0x2222222222; + scanPlan.id.subplanId = 0x3333333333; + scanPlan.type = QUERY_TYPE_SCAN; + scanPlan.level = 1; + scanPlan.execEpSet.numOfEps = 1; + scanPlan.pChildern = NULL; + scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); + + mergePlan.id.queryId = qId; + mergePlan.id.templateId = 0x4444444444; + mergePlan.id.subplanId = 0x5555555555; + mergePlan.type = QUERY_TYPE_MERGE; + mergePlan.level = 0; + mergePlan.execEpSet.numOfEps = 1; + mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES); + mergePlan.pParents = NULL; + + SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); + SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); + + taosArrayPush(mergePointer->pChildern, &scanPointer); + taosArrayPush(scanPointer->pParents, &mergePointer); + + taosArrayPush(dag->pSubplans, &merge); + taosArrayPush(dag->pSubplans, &scan); +} + +} + +TEST(testCase, normalCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + void *pJob = NULL; + SQueryDag dag = {0}; + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + mockBuildDag(&dag); + + code = scheduleExecJob(mockPointer, qnodeList, &dag, &pJob); + ASSERT_EQ(code, 0); +} + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + + diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f892b4d8c0..cb231e15a0 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1120,11 +1120,11 @@ SysNameInfo taosGetSysNameInfo() { struct utsname uts; if (!uname(&uts)) { - info.sysname = strdup(uts.sysname); - info.nodename = strdup(uts.nodename); - info.release = strdup(uts.release); - info.version = strdup(uts.version); - info.machine = strdup(uts.machine); + tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname)); + tstrncpy(info.nodename, uts.nodename, sizeof(info.nodename)); + tstrncpy(info.release, uts.release, sizeof(info.release)); + tstrncpy(info.version, uts.version, sizeof(info.version)); + tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } return info; diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 7c42afcc51..d343945a80 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -1,3 +1,4 @@ +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c") aux_source_directory(src UTIL_SRC) add_library(util STATIC ${UTIL_SRC}) target_include_directories( @@ -11,6 +12,4 @@ target_link_libraries( PUBLIC zlib PUBLIC lz4_static PUBLIC api -) - -CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c") +) \ No newline at end of file diff --git a/source/util/src/mallocator.c b/source/util/src/mallocator.c index 1819396ccd..a56fbfa597 100644 --- a/source/util/src/mallocator.c +++ b/source/util/src/mallocator.c @@ -16,6 +16,7 @@ #include "mallocator.h" /* ------------------------ HEAP ALLOCATOR ------------------------ */ +#if 0 typedef struct { size_t tusage; } SHeapAllocator; @@ -104,4 +105,5 @@ static size_t haUsage(SMemAllocator *pma) { return ((SHeapAllocator *)(pma->impl /* ------------------------ ARENA ALLOCATOR ------------------------ */ typedef struct { size_t usage; -} SArenaAllocator; \ No newline at end of file +} SArenaAllocator; +#endif \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 70a3dc622f..ffc59c5d69 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -130,6 +130,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE, "Value out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT, "Invalid tsc input") // mnode-common +TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") @@ -217,22 +218,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist // mnode-stable TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "Stable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS, "Too many stables") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB, "Invalid stable name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREAY_EXIST, "Column already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") +// mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") @@ -241,9 +240,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") - // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") @@ -322,6 +318,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit ha TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION, "One valid time range condition expected") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_INPUT, "invalid input") // grant @@ -500,6 +497,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready" TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error") +//scheduler +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") #ifdef TAOS_ERROR_C diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 6756af226f..f79bca1e4b 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -17,9 +17,8 @@ #include "os.h" void tdListInit(SList *list, int eleSize) { - list->eleSize = eleSize; - list->numOfEles = 0; - list->head = list->tail = NULL; + TD_DLIST_INIT(list); + listEleSize(list) = eleSize; } SList *tdListNew(int eleSize) { @@ -31,14 +30,11 @@ SList *tdListNew(int eleSize) { } void tdListEmpty(SList *list) { - SListNode *node = list->head; - while (node) { - list->head = node->next; + SListNode *node; + while ((node = TD_DLIST_HEAD(list)) != NULL) { + TD_DLIST_POP(list, node); free(node); - node = list->head; } - list->head = list->tail = 0; - list->numOfEles = 0; } void *tdListFree(SList *list) { @@ -50,40 +46,16 @@ void *tdListFree(SList *list) { return NULL; } -void tdListPrependNode(SList *list, SListNode *node) { - if (list->head == NULL) { - list->head = node; - list->tail = node; - } else { - node->next = list->head; - node->prev = NULL; - list->head->prev = node; - list->head = node; - } - list->numOfEles++; -} +void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, node); } -void tdListAppendNode(SList *list, SListNode *node) { - if (list->head == NULL) { - list->head = node; - list->tail = node; - } else { - node->prev = list->tail; - node->next = NULL; - list->tail->next = node; - list->tail = node; - } - - list->numOfEles++; -} +void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } int tdListPrepend(SList *list, void *data) { SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; - node->next = node->prev = NULL; memcpy((void *)(node->data), data, list->eleSize); - tdListPrependNode(list, node); + TD_DLIST_PREPEND(list, node); return 0; } @@ -93,73 +65,40 @@ int tdListAppend(SList *list, void *data) { if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); - tdListAppendNode(list, node); + TD_DLIST_APPEND(list, node); return 0; } SListNode *tdListPopHead(SList *list) { - if (list->head == NULL) return NULL; - SListNode *node = list->head; - if (node->next == NULL) { - list->head = NULL; - list->tail = NULL; - } else { - list->head = node->next; + SListNode *node; + + node = TD_DLIST_HEAD(list); + + if (node) { + TD_DLIST_POP(list, node); } - list->numOfEles--; - node->next = NULL; - node->prev = NULL; + return node; } SListNode *tdListPopTail(SList *list) { - if (list->tail == NULL) return NULL; - SListNode *node = list->tail; - if (node->prev == NULL) { - list->head = NULL; - list->tail = NULL; - } else { - list->tail = node->prev; + SListNode *node; + + node = TD_DLIST_TAIL(list); + if (node) { + TD_DLIST_POP(list, node); } - list->numOfEles--; - node->next = node->prev = NULL; + return node; } -SListNode *tdListGetHead(SList *list) { - if (list == NULL || list->numOfEles == 0) { - return NULL; - } +SListNode *tdListGetHead(SList *list) { return TD_DLIST_HEAD(list); } - return list->head; -} - -SListNode *tsListGetTail(SList *list) { - if (list == NULL || list->numOfEles == 0) { - return NULL; - } - - return list->tail; -} +SListNode *tsListGetTail(SList *list) { return TD_DLIST_TAIL(list); } SListNode *tdListPopNode(SList *list, SListNode *node) { - if (list->head == node) { - list->head = node->next; - } - if (list->tail == node) { - list->tail = node->prev; - } - - if (node->prev != NULL) { - node->prev->next = node->next; - } - if (node->next != NULL) { - node->next->prev = node->prev; - } - list->numOfEles--; - node->next = node->prev = NULL; - + TD_DLIST_POP(list, node); return node; } @@ -174,19 +113,19 @@ void tdListMove(SList *src, SList *dst) { void tdListDiscard(SList *list) { if (list) { - list->head = list->tail = NULL; - list->numOfEles = 0; + listHead(list) = listTail(list) = NULL; + listNEles(list) = 0; } } -void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); } +void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, listEleSize(list)); } void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction) { pIter->direction = direction; if (direction == TD_LIST_FORWARD) { - pIter->next = list->head; + pIter->next = listHead(list); } else { - pIter->next = list->tail; + pIter->next = listTail(list); } } @@ -194,9 +133,9 @@ SListNode *tdListNext(SListIter *pIter) { SListNode *node = pIter->next; if (node == NULL) return NULL; if (pIter->direction == TD_LIST_FORWARD) { - pIter->next = node->next; + pIter->next = TD_DLIST_NODE_NEXT(node); } else { - pIter->next = node->prev; + pIter->next = TD_DLIST_NODE_PREV(node); } return node; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 4a621d47c0..b2f8eda474 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -183,7 +183,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pSql->fp = fp; if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { @@ -265,7 +265,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { } return; - } else if (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { + } else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { // in case of show command, return no data (*pSql->fetchFp)(param, pSql, 0); } else { @@ -273,7 +273,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { } } else { // current query is not completed, continue retrieve from node if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 0b4f399a1f..dcfb2d6a87 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -322,7 +322,7 @@ TAOS_ROW tscFetchRow(void *param) { // current data set are exhausted, fetch more data from node if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE || + (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index caa334aaed..7a754ee698 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -539,7 +539,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) { if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || @@ -2749,7 +2749,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE) || + if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && @@ -3174,7 +3174,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -3192,7 +3192,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; @@ -3214,7 +3214,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; tscKeepConn[TSDB_SQL_SHOW] = 1; - tscKeepConn[TSDB_SQL_RETRIEVE] = 1; + tscKeepConn[TSDB_SQL_RETRIEVE_MNODE] = 1; tscKeepConn[TSDB_SQL_SELECT] = 1; tscKeepConn[TSDB_SQL_FETCH] = 1; tscKeepConn[TSDB_SQL_HB] = 1; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 5fdaad0d66..ab1fffd5a2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -460,7 +460,7 @@ static bool needToFetchNewBlock(SSqlObj* pSql) { SSqlCmd *pCmd = &pSql->cmd; return (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE || + (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || @@ -582,10 +582,10 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || - cmd == TSDB_SQL_RETRIEVE || + cmd == TSDB_SQL_RETRIEVE_MNODE || cmd == TSDB_SQL_FETCH)) { pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; tscDebug("0x%"PRIx64" send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql->self, sqlCmd[pCmd->command]); tscBuildAndSendRequest(pSql, NULL); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 275042a238..24a6377a73 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1670,7 +1670,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { pSql1->fp = joinRetrieveFinalResCallback; if (pCmd1->command < TSDB_SQL_LOCAL) { - pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } tscBuildAndSendRequest(pSql1, NULL); diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt deleted file mode 100644 index fdf58d5ae1..0000000000 --- a/src/kit/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) -ADD_SUBDIRECTORY(taospack) diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt deleted file mode 100644 index bf2bbca14d..0000000000 --- a/src/kit/shell/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(inc) - -IF (TD_LINUX) - AUX_SOURCE_DIRECTORY(./src SRC) - LIST(REMOVE_ITEM SRC ./src/shellWindows.c) - LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) - ADD_EXECUTABLE(shell ${SRC}) - -IF (TD_LINUX_64 AND JEMALLOC_ENABLED) - ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) - SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") -ELSE () - SET(LINK_JEMALLOC "") -ENDIF () - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(shell taos_static lua ${LINK_JEMALLOC}) - ELSE () - TARGET_LINK_LIBRARIES(shell taos lua ${LINK_JEMALLOC}) - ENDIF () - - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) -ELSEIF (TD_WINDOWS) - LIST(APPEND SRC ./src/shellEngine.c) - LIST(APPEND SRC ./src/shellMain.c) - LIST(APPEND SRC ./src/shellWindows.c) - ADD_EXECUTABLE(shell ${SRC}) - TARGET_LINK_LIBRARIES(shell taos_static) - - IF (TD_POWER) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) - ELSE () - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) - ENDIF () -ELSEIF (TD_DARWIN) - LIST(APPEND SRC ./src/shellEngine.c) - LIST(APPEND SRC ./src/shellMain.c) - LIST(APPEND SRC ./src/shellDarwin.c) - LIST(APPEND SRC ./src/shellCommand.c) - LIST(APPEND SRC ./src/shellImport.c) - LIST(APPEND SRC ./src/shellCheck.c) - ADD_EXECUTABLE(shell ${SRC}) - # linking with dylib - TARGET_LINK_LIBRARIES(shell taos) - # linking taos statically - # TARGET_LINK_LIBRARIES(shell taos_static) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) -ENDIF () - diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e21905af3b..966eb94354 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,15 +1,4 @@ -# generate debug version: -# mkdir debug; cd debug; cmake -DCMAKE_BUILD_TYPE=Debug .. -# generate release version: -# mkdir release; cd release; cmake -DCMAKE_BUILD_TYPE=Release .. - -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -SET(CMAKE_C_STANDARD 11) -SET(CMAKE_VERBOSE_MAKEFILE ON) - -ADD_SUBDIRECTORY(examples/c) +#ADD_SUBDIRECTORY(examples/c) ADD_SUBDIRECTORY(tsim) -ADD_SUBDIRECTORY(test/c) -ADD_SUBDIRECTORY(comparisonTest/tdengine) +#ADD_SUBDIRECTORY(test/c) +#ADD_SUBDIRECTORY(comparisonTest/tdengine) diff --git a/tests/script/general/user/basic1.sim b/tests/script/general/user/basic1.sim index 3670c1ddb0..d4a663c096 100644 --- a/tests/script/general/user/basic1.sim +++ b/tests/script/general/user/basic1.sim @@ -5,7 +5,7 @@ sql connect print =============== show users sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi @@ -21,7 +21,7 @@ sql_error drop account root print =============== create user1 sql create user user1 PASS 'user1' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi @@ -33,7 +33,7 @@ print $data30 $data31 $data32 print =============== create user2 sql create user user2 PASS 'user2' sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -46,7 +46,7 @@ print $data40 $data41 $data42 print =============== drop user1 sql drop user user1 sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi @@ -62,7 +62,7 @@ system sh/exec.sh -n dnode1 -s start print =============== show users sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cde27d7dc3..03ce1c2888 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -1,5 +1,8 @@ #!/bin/bash +set +e +#set -x + echo "Executing deploy.sh" if [ $# != 4 ]; then @@ -50,12 +53,12 @@ else fi if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3` else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2` fi -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 80b8cda428..d1572bb513 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -8,6 +8,9 @@ # exit 1 # fi +set +e +#set -x + UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` @@ -62,16 +65,16 @@ else fi if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3` else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2` fi -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/bin +EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon CFG_DIR=$NODE_DIR/cfg LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data diff --git a/tests/script/test.sh b/tests/script/test.sh index f2dc578987..88ed759296 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -22,9 +22,6 @@ do f) FILE_NAME=$OPTARG ;; - a) - ASYNC=1 - ;; v) VALGRIND=1 ;; @@ -60,32 +57,22 @@ else fi if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3` else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2` fi -BUILD_DIR=$TOP_DIR/$BIN_DIR/build +BUILD_DIR=$TOP_DIR/$BIN_DIR SIM_DIR=$TOP_DIR/sim -if [ $ASYNC -eq 0 ]; then - PROGRAM=$BUILD_DIR/bin/tsim -else - PROGRAM="$BUILD_DIR/bin/tsim -a" -fi - +PROGRAM=$BUILD_DIR/tests/tsim/tsim PRG_DIR=$SIM_DIR/tsim CFG_DIR=$PRG_DIR/cfg LOG_DIR=$PRG_DIR/log DATA_DIR=$PRG_DIR/data - -ARBITRATOR_PRG_DIR=$SIM_DIR/arbitrator -ARBITRATOR_LOG_DIR=$ARBITRATOR_PRG_DIR/log - - chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" echo "Start TDengine Testing Case ..." @@ -96,12 +83,10 @@ echo "CFG_DIR : $CFG_DIR" rm -rf $LOG_DIR rm -rf $CFG_DIR -rm -rf $ARBITRATOR_LOG_DIR mkdir -p $PRG_DIR mkdir -p $LOG_DIR mkdir -p $CFG_DIR -mkdir -p $ARBITRATOR_LOG_DIR TAOS_CFG=$PRG_DIR/cfg/taos.cfg touch -f $TAOS_CFG @@ -115,7 +100,7 @@ echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "serverPort 7100" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "scriptDir ${CODE_DIR}/../script" >> $TAOS_CFG +echo "scriptDir ${CODE_DIR}" >> $TAOS_CFG echo "numOfLogLines 100000000" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG @@ -141,7 +126,6 @@ if [ -n "$FILE_NAME" ]; then else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME $PROGRAM -c $CFG_DIR -f $FILE_NAME -# valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME fi else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt index 50b42941af..81737809d9 100644 --- a/tests/tsim/CMakeLists.txt +++ b/tests/tsim/CMakeLists.txt @@ -1,9 +1,14 @@ -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) -INCLUDE_DIRECTORIES(inc) - -AUX_SOURCE_DIRECTORY(src SRC) -ADD_EXECUTABLE(tsim ${SRC}) -TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson) +aux_source_directory(src TSIM_SRC) +add_executable(tsim ${TSIM_SRC}) +target_link_libraries( + tsim + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os + PUBLIC cjson +) +target_include_directories( + tsim + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 39c6f6ac36..4aa3c62a80 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef __SIM_H__ -#define __SIM_H__ +#ifndef _TD_SIM_H_ +#define _TD_SIM_H_ #include #include @@ -102,18 +102,18 @@ typedef struct _cmd_t { int16_t cmdno; int16_t nlen; char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int32_t); - bool (*executeCmd)(struct _script_t *script, char *option); + bool (*parseCmd)(char *, struct _cmd_t *, int32_t); + bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; typedef struct { int16_t cmdno; - int16_t jump; // jump position - int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag - // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ - int16_t lineNum; // correspodning line number in original file - int32_t optionOffset;// relative option offset + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file + int32_t optionOffset; // relative option offset } SCmdLine; typedef struct _var_t { @@ -123,24 +123,24 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int32_t type; - bool killed; - void * taos; - char rows[12]; // number of rows data retrieved - char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results - char system_exit_code[12]; - char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - int32_t varLen; - int32_t linePos; // current cmd position - int32_t numOfLines; // number of lines in the script - int32_t bgScriptLen; - char fileName[MAX_FILE_NAME_LEN]; // script file name - char error[MAX_ERROR_LEN]; - char * optionBuffer; - SCmdLine *lines; // command list - SVariable variables[MAX_VAR_LEN]; - pthread_t bgPid; - char auth[128]; + int32_t type; + bool killed; + void *taos; + char rows[12]; // number of rows data retrieved + char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results + char system_exit_code[12]; + char system_ret_content[MAX_SYSTEM_RESULT_LEN]; + int32_t varLen; + int32_t linePos; // current cmd position + int32_t numOfLines; // number of lines in the script + int32_t bgScriptLen; + char fileName[MAX_FILE_NAME_LEN]; // script file name + char error[MAX_ERROR_LEN]; + char *optionBuffer; + SCmdLine *lines; // command list + SVariable variables[MAX_VAR_LEN]; + pthread_t bgPid; + char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; } SScript; @@ -150,16 +150,15 @@ extern int32_t simScriptPos; extern int32_t simScriptSucced; extern int32_t simDebugFlag; extern char tsScriptDir[]; -extern bool simAsyncQuery; extern bool abortExecution; SScript *simParseScript(char *fileName); SScript *simProcessCallOver(SScript *script); -void * simExecuteScript(void *script); +void *simExecuteScript(void *script); void simInitsimCmdList(); bool simSystemInit(); void simSystemCleanUp(); -char * simGetVariable(SScript *script, char *varName, int32_t varLen); +char *simGetVariable(SScript *script, char *varName, int32_t varLen); bool simExecuteExpCmd(SScript *script, char *option); bool simExecuteTestCmd(SScript *script, char *option); bool simExecuteGotoCmd(SScript *script, char *option); @@ -178,4 +177,4 @@ bool simExecuteLineInsertCmd(SScript *script, char *option); bool simExecuteLineInsertErrorCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); -#endif \ No newline at end of file +#endif /*_TD_SIM_H_*/ \ No newline at end of file diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index ef7d8e5ce7..56ee90aadb 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef __SIM_PARSE_H__ -#define __SIM_PARSE_H__ +#ifndef _TD_SIM_PARSE_H_ +#define _TD_SIM_PARSE_H_ #define MAX_NUM_CMD 64 #define MAX_NUM_LABLES 100 @@ -40,10 +40,10 @@ typedef struct { /* block definition */ typedef struct { - char top; /* the number of blocks stacked */ - char type[MAX_NUM_BLOCK]; /* the block type */ - int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ char numJump[MAX_NUM_BLOCK]; int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ @@ -52,4 +52,4 @@ typedef struct { bool simParseExpression(char *token, int32_t lineNum); -#endif \ No newline at end of file +#endif /*_TD_SIM_PARSE_H_*/ \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 7a75dd7d85..453d653919 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -14,18 +14,18 @@ */ #define _DEFAULT_SOURCE -#include "../../../include/client/taos.h" #include "cJSON.h" #include "os.h" #include "sim.h" +#include "taos.h" #include "taoserror.h" #include "tglobal.h" +#include "ttypes.h" #include "tutil.h" -#undef TAOS_MEM_CHECK void simLogSql(char *sql, bool useSharp) { static FILE *fp = NULL; - char filename[256]; + char filename[256]; sprintf(filename, "%s/sim.sql", tsScriptDir); if (fp == NULL) { fp = fopen(filename, "w"); @@ -74,7 +74,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return "null"; } - char * keyName; + char *keyName; int32_t keyLen; paGetToken(varName + 6, &keyName, &keyLen); @@ -91,7 +91,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return "null"; } - char * keyName; + char *keyName; int32_t keyLen; paGetToken(varName + 7, &keyName, &keyLen); @@ -144,7 +144,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } int32_t simExecuteExpression(SScript *script, char *exp) { - char * op1, *op2, *var1, *var2, *var3, *rest; + char *op1, *op2, *var1, *var2, *var3, *rest; int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; char t0[1024], t1[1024], t2[1024], t3[2048]; int32_t result; @@ -302,10 +302,10 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { } void simReplaceShToBat(char *dst) { - char* sh = strstr(dst, ".sh"); + char *sh = strstr(dst, ".sh"); if (sh != NULL) { int32_t dstLen = (int32_t)strlen(dst); - char *end = dst + dstLen; + char *end = dst + dstLen; *(end + 1) = 0; for (char *p = end; p >= sh; p--) { @@ -436,7 +436,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) { } void simVisuallizeOption(SScript *script, char *src, char *dst) { - char * var, *token, *value; + char *var, *token, *value; int32_t dstLen, srcLen, tokenLen; dst[0] = 0, dstLen = 0; @@ -466,10 +466,6 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { strcpy(dst + dstLen, src); } -void simCloseRestFulConnect(SScript *script) { - memset(script->auth, 0, sizeof(script->auth)); -} - void simCloseNativeConnect(SScript *script) { if (script->taos == NULL) return; @@ -479,168 +475,7 @@ void simCloseNativeConnect(SScript *script) { script->taos = NULL; } -void simCloseTaosdConnect(SScript *script) { - if (simAsyncQuery) { - simCloseRestFulConnect(script); - } else { - simCloseNativeConnect(script); - } -} -// {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} -// {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1} -// {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10} -int32_t simParseHttpCommandResult(SScript *script, char *command) { - cJSON* root = cJSON_Parse(command); - if (root == NULL) { - simError("script:%s, failed to parse json, response:%s", script->fileName, command); - return -1; - } - - cJSON *status = cJSON_GetObjectItem(root, "status"); - if (status == NULL) { - simError("script:%s, failed to parse json, status is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - - if (status->valuestring == NULL || strlen(status->valuestring) == 0) { - simError("script:%s, failed to parse json, status value is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - - if (strcmp(status->valuestring, "succ") != 0) { - cJSON *code = cJSON_GetObjectItem(root, "code"); - if (code == NULL) { - simError("script:%s, failed to parse json, code is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - int32_t retcode = (int32_t)code->valueint; - if (retcode != 1017) { - simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, - command); - cJSON_Delete(root); - return retcode; - } else { - simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName, - status->valuestring, retcode, command); - cJSON_Delete(root); - return 0; - } - } - - cJSON *desc = cJSON_GetObjectItem(root, "desc"); - if (desc != NULL) { - if (desc->valuestring == NULL || strlen(desc->valuestring) == 0) { - simError("script:%s, failed to parse json, desc value is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - strcpy(script->auth, desc->valuestring); - cJSON_Delete(root); - return 0; - } - - cJSON *data = cJSON_GetObjectItem(root, "data"); - if (data == NULL) { - simError("script:%s, failed to parse json, data is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - - int32_t rowsize = cJSON_GetArraySize(data); - if (rowsize < 0) { - simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command); - cJSON_Delete(root); - return -1; - } - - int32_t rowIndex = 0; - sprintf(script->rows, "%d", rowsize); - for (int32_t r = 0; r < rowsize; ++r) { - cJSON *row = cJSON_GetArrayItem(data, r); - if (row == NULL) continue; - if (rowIndex++ >= 10) break; - - int32_t colsize = cJSON_GetArraySize(row); - if (colsize < 0) { - break; - } - - colsize = MIN(10, colsize); - for (int32_t c = 0; c < colsize; ++c) { - cJSON *col = cJSON_GetArrayItem(row, c); - if (col->valuestring != NULL) { - strcpy(script->data[r][c], col->valuestring); - } else { - if (col->numberstring[0] == 0) { - strcpy(script->data[r][c], "null"); - } else { - strcpy(script->data[r][c], col->numberstring); - } - } - } - } - - return 0; -} - -int32_t simExecuteRestFulCommand(SScript *script, char *command) { - char buf[5000] = {0}; - sprintf(buf, "%s 2>/dev/null", command); - - FILE *fp = popen(buf, "r"); - if (fp == NULL) { - simError("failed to execute %s", buf); - return -1; - } - - int32_t mallocSize = 2000; - int32_t alreadyReadSize = 0; - char * content = malloc(mallocSize); - - while (!feof(fp)) { - int32_t availSize = mallocSize - alreadyReadSize; - int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp); - if (len >= availSize) { - alreadyReadSize += len; - mallocSize *= 2; - content = realloc(content, mallocSize); - } - } - - pclose(fp); - - return simParseHttpCommandResult(script, content); -} - -bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { - char command[4096]; - sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass); - - bool success = false; - for (int32_t attempt = 0; attempt < 10; ++attempt) { - success = simExecuteRestFulCommand(script, command) == 0; - if (!success) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), - attempt); - taosMsleep(1000); - } else { - simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); - break; - } - } - - if (!success) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, - taos_errstr(NULL)); - return false; - } - - simDebug("script:%s, connect taosd successed, auth:%p", script->fileName, script->auth); - return true; -} +void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); } bool simCreateNativeConnect(SScript *script, char *user, char *pass) { simCloseTaosdConnect(script); @@ -651,7 +486,7 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { return false; } - taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort); + taos = taos_connect(NULL, user, pass, NULL, 0); if (taos == NULL) { simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt); @@ -675,8 +510,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { } bool simCreateTaosdConnect(SScript *script, char *rest) { - char * user = TSDB_DEFAULT_USER; - char * token; + char *user = TSDB_DEFAULT_USER; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); rest = paGetToken(rest, &token, &tokenLen); @@ -684,18 +519,14 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { user = token; } - if (simAsyncQuery) { - return simCreateRestFulConnect(script, user, TSDB_DEFAULT_PASS); - } else { - return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); - } + return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); } bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { char timeStr[30] = {0}; time_t tt; struct tm *tp; - SCmdLine * line = &script->lines[script->linePos]; + SCmdLine *line = &script->lines[script->linePos]; int32_t ret = -1; TAOS_RES *pSql = NULL; @@ -710,7 +541,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); - if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { + if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); ret = 0; @@ -756,7 +587,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { while ((row = taos_fetch_row(pSql))) { if (numOfRows < MAX_QUERY_ROW_NUM) { TAOS_FIELD *fields = taos_fetch_fields(pSql); - int32_t * length = taos_fetch_lengths(pSql); + int32_t *length = taos_fetch_lengths(pSql); for (int32_t i = 0; i < num_fields; i++) { char *value = NULL; @@ -780,7 +611,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%d", *((int8_t *)row[i])); break; case TSDB_DATA_TYPE_UTINYINT: - sprintf(value, "%u", *((uint8_t*)row[i])); + sprintf(value, "%u", *((uint8_t *)row[i])); break; case TSDB_DATA_TYPE_SMALLINT: sprintf(value, "%d", *((int16_t *)row[i])); @@ -846,7 +677,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else if (precision == TSDB_TIME_PRECISION_MICRO) { sprintf(value, "%s.%06d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000)); } else { - sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); + sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); } break; @@ -877,43 +708,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { return true; } -bool simExecuteRestFulSqlCommand(SScript *script, char *rest) { - SCmdLine *line = &script->lines[script->linePos]; - char command[4096]; - sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest); - - int32_t ret = -1; - for (int32_t attempt = 0; attempt < 10; ++attempt) { - ret = simExecuteRestFulCommand(script, command); - if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, - tstrerror(ret)); - ret = 0; - break; - } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret); - - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - taosMsleep(1000); - } else { - break; - } - } - - if (ret) { - sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d", line->lineNum, rest, ret); - return false; - } - - script->linePos++; - return true; -} - bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { - char buf[3000]; + char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; simVisuallizeOption(script, rest, buf); @@ -935,7 +731,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { return true; } - if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) { + if (script->taos == NULL) { if (!simCreateTaosdConnect(script, "connect root")) { if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -951,11 +747,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { return true; } - if (simAsyncQuery) { - return simExecuteRestFulSqlCommand(script, rest); - } else { - return simExecuteNativeSqlCommand(script, rest, isSlow); - } + return simExecuteNativeSqlCommand(script, rest, isSlow); } bool simExecuteSqlCmd(SScript *script, char *rest) { @@ -1010,7 +802,7 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) { } bool simExecuteSqlErrorCmd(SScript *script, char *rest) { - char buf[3000]; + char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; simVisuallizeOption(script, rest, buf); @@ -1032,7 +824,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) { + if (script->taos == NULL) { if (!simCreateTaosdConnect(script, "connect root")) { if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -1048,17 +840,9 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - int32_t ret; - TAOS_RES *pSql = NULL; - if (simAsyncQuery) { - char command[4096]; - sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest); - ret = simExecuteRestFulCommand(script, command); - } else { - pSql = taos_query(script->taos, rest); - ret = taos_errno(pSql); - taos_free_result(pSql); - } + TAOS_RES *pSql = pSql = taos_query(script->taos, rest); + int32_t ret = taos_errno(pSql); + taos_free_result(pSql); if (ret != TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, @@ -1083,15 +867,19 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); - char * lines[] = {rest}; + char *lines[] = {rest}; +#if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif if (ret == TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); script->linePos++; return true; } else { - sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, - ret & 0XFFFF, tstrerror(ret)); + sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); return false; } } @@ -1106,15 +894,20 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); - char * lines[] = {rest}; + char *lines[] = {rest}; +#if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif if (ret == TSDB_CODE_SUCCESS) { - sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest); + sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, + script->taos, rest); script->linePos++; return false; } else { - simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, - ret & 0XFFFF, tstrerror(ret)); + simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); return true; } } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 7d74c62c7d..19d23daace 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -15,19 +15,17 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tglobal.h" #include "sim.h" -#undef TAOS_MEM_CHECK +#include "tglobal.h" -bool simAsyncQuery = false; bool simExecSuccess = false; bool abortExecution = false; void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); abortExecution = true; -// runningScript->killed = true; -// exit(1); + // runningScript->killed = true; + // exit(1); } int32_t main(int32_t argc, char *argv[]) { @@ -38,8 +36,6 @@ int32_t main(int32_t argc, char *argv[]) { tstrncpy(configDir, argv[++i], 128); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { strcpy(scriptFile, argv[++i]); - } else if (strcmp(argv[i], "-a") == 0) { - simAsyncQuery = true; } else { printf("usage: %s [options] \n", argv[0]); printf(" [-c config]: config directory, default is: %s\n", configDir); diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 1acdcd2ac6..0dfd5f4f3e 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -60,9 +60,9 @@ #define _DEFAULT_SOURCE #include "os.h" #include "sim.h" -#include "simParse.h" #include "tutil.h" -#undef TAOS_MEM_CHECK + +#include "simParse.h" static SCommand *cmdHashList[MAX_NUM_CMD]; static SCmdLine cmdLine[MAX_CMD_LINES]; @@ -177,11 +177,11 @@ SScript *simBuildScriptObj(char *fileName) { } SScript *simParseScript(char *fileName) { - FILE * fd; + FILE *fd; int32_t tokenLen, lineNum = 0; char buffer[MAX_LINE_LEN], name[128], *token, *rest; SCommand *pCmd; - SScript * script; + SScript *script; if ((fileName[0] == '.') || (fileName[0] == '/')) { strcpy(name, fileName); @@ -252,7 +252,7 @@ SScript *simParseScript(char *fileName) { } int32_t simCheckExpression(char *exp) { - char * op1, *op2, *op, *rest; + char *op1, *op2, *op, *rest; int32_t op1Len, op2Len, opLen; rest = paGetToken(exp, &op1, &op1Len); @@ -336,7 +336,7 @@ bool simParseExpression(char *token, int32_t lineNum) { } bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * ret; + char *ret; int32_t expLen; expLen = simCheckExpression(rest); @@ -502,7 +502,7 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -525,7 +525,7 @@ bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -666,7 +666,7 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { void simCheckSqlOption(char *rest) { int32_t valueLen; - char * value, *xpos; + char *value, *xpos; xpos = strstr(rest, " -x"); // need a blank if (xpos) { @@ -750,7 +750,7 @@ bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; @@ -769,7 +769,7 @@ bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; @@ -788,7 +788,7 @@ bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -811,7 +811,7 @@ bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -838,7 +838,7 @@ bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } -bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) { +bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; rest++; @@ -854,7 +854,7 @@ bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) { return true; } -bool simParseLineInsertErrorCmd(char* rest, SCommand* pCmd, int32_t lineNum) { +bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; rest++; diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 65612930ef..cb61e6b814 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -14,15 +14,15 @@ */ #define _DEFAULT_SOURCE -#include "../../../include/client/taos.h" #include "os.h" #include "sim.h" +#include "taos.h" #include "taoserror.h" #include "tglobal.h" -#include "tsocket.h" #include "ttimer.h" #include "tutil.h" -#undef TAOS_MEM_CHECK +#include "tglobal.h" +#include "tconfig.h" SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; @@ -81,10 +81,11 @@ char *simParseHostName(char *varName) { } bool simSystemInit() { - if (taos_init()) { - return false; - } taosGetFqdn(simHostName); + + taosInitGlobalCfg(); + taosReadCfgFromFile(); + simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); return true; @@ -171,7 +172,7 @@ void *simExecuteScript(void *inputScript) { } } else { SCmdLine *line = &script->lines[script->linePos]; - char * option = script->optionBuffer + line->optionOffset; + char *option = script->optionBuffer + line->optionOffset; simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); SCommand *cmd = &simCmdList[line->cmdno]; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000..629677c9a5 --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(shell) \ No newline at end of file diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt new file mode 100644 index 0000000000..def421c212 --- /dev/null +++ b/tools/shell/CMakeLists.txt @@ -0,0 +1,18 @@ +aux_source_directory(src SHELL_SRC) +list(REMOVE_ITEM SHELL_SRC src/shellWindows.c) +list(REMOVE_ITEM SHELL_SRC src/shellDarwin.c) + +add_executable(shell ${SHELL_SRC}) +target_link_libraries( + shell + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os +) +target_include_directories( + shell + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) diff --git a/src/kit/shell/inc/shell.h b/tools/shell/inc/shell.h similarity index 95% rename from src/kit/shell/inc/shell.h rename to tools/shell/inc/shell.h index 10221a409a..cf31e9d9d9 100644 --- a/src/kit/shell/inc/shell.h +++ b/tools/shell/inc/shell.h @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#ifndef __SHELL__ -#define __SHELL__ +#ifndef _TD_SHELL_H_ +#define _TD_SHELL_H_ -#include "../../../../include/client/taos.h" -#include "stdbool.h" +#include "os.h" + +#include "taos.h" #include "taosdef.h" -#include "tsclient.h" #define MAX_USERNAME_SIZE 64 #define MAX_DBNAME_SIZE 64 diff --git a/src/kit/shell/inc/shellCommand.h b/tools/shell/inc/shellCommand.h similarity index 96% rename from src/kit/shell/inc/shellCommand.h rename to tools/shell/inc/shellCommand.h index 6e4d3e382e..49f7dc0133 100644 --- a/src/kit/shell/inc/shellCommand.h +++ b/tools/shell/inc/shellCommand.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef __COMMAND_STRUCT__ -#define __COMMAND_STRUCT__ +#ifndef _TD_SHELL_COMMAND_H_ +#define _TD_SHELL_COMMAND_H_ #include "shell.h" diff --git a/src/kit/shell/src/shellCheck.c b/tools/shell/src/backup/shellCheck.c similarity index 100% rename from src/kit/shell/src/shellCheck.c rename to tools/shell/src/backup/shellCheck.c diff --git a/src/kit/shell/src/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c similarity index 100% rename from src/kit/shell/src/shellDarwin.c rename to tools/shell/src/backup/shellDarwin.c diff --git a/src/kit/shell/src/shellImport.c b/tools/shell/src/backup/shellImport.c similarity index 100% rename from src/kit/shell/src/shellImport.c rename to tools/shell/src/backup/shellImport.c diff --git a/src/kit/shell/src/shellWindows.c b/tools/shell/src/backup/shellWindows.c similarity index 100% rename from src/kit/shell/src/shellWindows.c rename to tools/shell/src/backup/shellWindows.c diff --git a/src/kit/shell/src/tnettest.c b/tools/shell/src/backup/tnettest.c similarity index 100% rename from src/kit/shell/src/tnettest.c rename to tools/shell/src/backup/tnettest.c diff --git a/src/kit/shell/inc/tnettest.h b/tools/shell/src/backup/tnettest.h similarity index 100% rename from src/kit/shell/inc/tnettest.h rename to tools/shell/src/backup/tnettest.h diff --git a/src/kit/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c similarity index 99% rename from src/kit/shell/src/shellCommand.c rename to tools/shell/src/shellCommand.c index 67e0c94989..cf0ceded38 100644 --- a/src/kit/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -19,6 +19,8 @@ #include "shell.h" #include "shellCommand.h" +#include + extern int wcwidth(wchar_t c); extern int wcswidth(const wchar_t *s, size_t n); typedef struct { diff --git a/src/kit/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c similarity index 85% rename from src/kit/shell/src/shellEngine.c rename to tools/shell/src/shellEngine.c index efc37403b4..1ad61ee2b0 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -21,13 +21,14 @@ #include "os.h" #include "shell.h" #include "shellCommand.h" -#include "tutil.h" #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" -#include "tsclient.h" +#include "ttypes.h" +#include "tutil.h" #include +#include /**************** Global variables ****************/ #ifdef _TD_POWER_ @@ -58,7 +59,7 @@ SShellHistory history; #define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 extern int32_t tsMaxBinaryDisplayWidth; -extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); /* * FUNCTION: Initialize the shell. @@ -89,12 +90,6 @@ TAOS *shellInit(SShellArguments *_args) { _args->user = TSDB_DEFAULT_USER; } - if (taos_init()) { - printf("failed to init taos\n"); - fflush(stdout); - return NULL; - } - // Connect to the database. TAOS *con = NULL; if (_args->auth == NULL) { @@ -127,6 +122,7 @@ TAOS *shellInit(SShellArguments *_args) { exit(EXIT_SUCCESS); } +#if 0 #ifndef WINDOWS if (_args->dir[0] != 0) { source_dir(con, _args); @@ -139,12 +135,13 @@ TAOS *shellInit(SShellArguments *_args) { taos_close(con); exit(EXIT_SUCCESS); } +#endif #endif return con; } -static bool isEmptyCommand(const char* cmd) { +static bool isEmptyCommand(const char *cmd) { for (char c = *cmd++; c != 0; c = *cmd++) { if (c != ' ' && c != '\t' && c != ';') { return false; @@ -153,7 +150,6 @@ static bool isEmptyCommand(const char* cmd) { return true; } - static int32_t shellRunSingleCommand(TAOS *con, char *command) { /* If command is empty just return */ if (isEmptyCommand(command)) { @@ -176,10 +172,11 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) { return 0; } - if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { + if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", + REG_EXTENDED | REG_ICASE)) { strtok(command, " \t"); strtok(NULL, " \t"); - char* p = strtok(NULL, " \t"); + char *p = strtok(NULL, " \t"); if (strcasecmp(p, "default") == 0) { tsMaxBinaryDisplayWidth = DEFAULT_MAX_BINARY_DISPLAY_WIDTH; } else { @@ -202,8 +199,7 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) { return 0; } - -int32_t shellRunCommand(TAOS* con, char* command) { +int32_t shellRunCommand(TAOS *con, char *command) { /* If command is empty just return */ if (isEmptyCommand(command)) { return 0; @@ -255,7 +251,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { if (c == '\\') { if (quote != 0 && (*command == '_' || *command == '\\')) { - //DO nothing + // DO nothing } else { esc = true; continue; @@ -284,21 +280,22 @@ int32_t shellRunCommand(TAOS* con, char* command) { return shellRunSingleCommand(con, cmd); } - void freeResultWithRid(int64_t rid) { +#if 0 SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); if(pSql){ taos_free_result(pSql); taosReleaseRef(tscObjRef, rid); } +#endif } void shellRunCommandOnServer(TAOS *con, char command[]) { int64_t st, et; wordexp_t full_path; - char * sptr = NULL; - char * cptr = NULL; - char * fname = NULL; + char *sptr = NULL; + char *cptr = NULL; + char *fname = NULL; bool printMode = false; if ((sptr = strstr(command, ">>")) != NULL) { @@ -327,7 +324,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); - TAOS_RES* pSql = taos_query_h(con, command, &result); + TAOS_RES *pSql = taos_query(con, command); if (taos_errno(pSql)) { taos_error(pSql, st); return; @@ -344,7 +341,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } - if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands + TAOS_FIELD* pFields = taos_fetch_fields(pSql); + if (pFields != NULL) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); @@ -405,14 +403,13 @@ int regex_match(const char *s, const char *reg, int cflags) { return 0; } - -static char* formatTimestamp(char* buf, int64_t val, int precision) { +static char *formatTimestamp(char *buf, int64_t val, int precision) { if (args.is_raw_time) { sprintf(buf, "%" PRId64, val); return buf; } - time_t tt; + time_t tt; int32_t ms = 0; if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); @@ -425,13 +422,13 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { ms = val % 1000; } -/* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ #ifdef WINDOWS if (tt < 0) tt = 0; @@ -447,8 +444,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } } - struct tm* ptm = localtime(&tt); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm *ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); @@ -461,8 +458,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } - -static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) { +static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { fprintf(fp, "%s", TSDB_DATA_NULL_STR); return; @@ -498,7 +494,7 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_ fprintf(fp, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t*)val, precision); + formatTimestamp(buf, *(int64_t *)val, precision); fprintf(fp, "'%s'", buf); break; default: @@ -506,7 +502,7 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_ } } -static int dumpResultToFile(const char* fname, TAOS_RES* tres) { +static int dumpResultToFile(const char *fname, TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; @@ -519,7 +515,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { return -1; } - FILE* fp = fopen(full_path.we_wordv[0], "w"); + FILE *fp = fopen(full_path.we_wordv[0], "w"); if (fp == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); @@ -528,9 +524,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { wordfree(&full_path); - int num_fields = taos_num_fields(tres); + int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int precision = taos_result_precision(tres); for (int col = 0; col < num_fields; col++) { if (col > 0) { @@ -542,18 +538,18 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { int numOfRows = 0; do { - int32_t* length = taos_fetch_lengths(tres); + int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { fputc(',', fp); } - dumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision); + dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); } fputc('\n', fp); numOfRows++; row = taos_fetch_row(tres); - } while( row != NULL); + } while (row != NULL); result = 0; fclose(fp); @@ -561,14 +557,13 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { return numOfRows; } - static void shellPrintNChar(const char *str, int length, int width) { wchar_t tail[3]; - int pos = 0, cols = 0, totalCols = 0, tailLen = 0; + int pos = 0, cols = 0, totalCols = 0, tailLen = 0; while (pos < length) { wchar_t wc; - int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); + int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); if (bytes == 0) { break; } @@ -625,8 +620,7 @@ static void shellPrintNChar(const char *str, int length, int width) { } } - -static void printField(const char* val, TAOS_FIELD* field, int width, int32_t length, int precision) { +static void printField(const char *val, TAOS_FIELD *field, int width, int32_t length, int precision) { if (val == NULL) { int w = width; if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) { @@ -679,7 +673,7 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t*)val, precision); + formatTimestamp(buf, *(int64_t *)val, precision); printf("%s", buf); break; default: @@ -687,27 +681,26 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le } } - -bool isSelectQuery(TAOS_RES* tres) { +bool isSelectQuery(TAOS_RES *tres) { +#if 0 char *sql = tscGetSqlStr(tres); if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) { return true; } - +#endif return false; } - -static int verticalPrintResult(TAOS_RES* tres) { +static int verticalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int precision = taos_result_precision(tres); int maxColNameLen = 0; for (int col = 0; col < num_fields; col++) { @@ -719,7 +712,7 @@ static int verticalPrintResult(TAOS_RES* tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /*&& !tscIsQueryWithLimit(tres)*/) { resShowMaxNum = DEFAULT_RES_SHOW_NUM; } @@ -729,52 +722,52 @@ static int verticalPrintResult(TAOS_RES* tres) { if (numOfRows < resShowMaxNum) { printf("*************************** %d.row ***************************\n", numOfRows + 1); - int32_t* length = taos_fetch_lengths(tres); + int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { - TAOS_FIELD* field = fields + i; + TAOS_FIELD *field = fields + i; int padding = (int)(maxColNameLen - strlen(field->name)); printf("%*.s%s: ", padding, " ", field->name); - printField((const char*)row[i], field, 0, length[i], precision); + printField((const char *)row[i], field, 0, length[i], precision); putchar('\n'); } } else if (showMore) { - printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); - printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); - showMore = 0; + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); + showMore = 0; } numOfRows++; row = taos_fetch_row(tres); - } while(row != NULL); + } while (row != NULL); return numOfRows; } -static int calcColWidth(TAOS_FIELD* field, int precision) { +static int calcColWidth(TAOS_FIELD *field, int precision) { int width = (int)strlen(field->name); switch (field->type) { case TSDB_DATA_TYPE_BOOL: - return MAX(5, width); // 'false' + return MAX(5, width); // 'false' case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - return MAX(4, width); // '-127' + return MAX(4, width); // '-127' case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - return MAX(6, width); // '-32767' + return MAX(6, width); // '-32767' case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: - return MAX(11, width); // '-2147483648' + return MAX(11, width); // '-2147483648' case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: - return MAX(21, width); // '-9223372036854775807' + return MAX(21, width); // '-9223372036854775807' case TSDB_DATA_TYPE_FLOAT: return MAX(20, width); @@ -801,12 +794,13 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { return MAX(14, width); - } if (precision == TSDB_TIME_PRECISION_NANO) { + } + if (precision == TSDB_TIME_PRECISION_NANO) { return MAX(29, width); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - return MAX(26, width); // '2020-01-01 00:00:00.000000' + return MAX(26, width); // '2020-01-01 00:00:00.000000' } else { - return MAX(23, width); // '2020-01-01 00:00:00.000' + return MAX(23, width); // '2020-01-01 00:00:00.000' } default: @@ -816,13 +810,12 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { return 0; } - -static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) { +static void printHeader(TAOS_FIELD *fields, int *width, int num_fields) { int rowWidth = 0; for (int col = 0; col < num_fields; col++) { - TAOS_FIELD* field = fields + col; - int padding = (int)(width[col] - strlen(field->name)); - int left = padding / 2; + TAOS_FIELD *field = fields + col; + int padding = (int)(width[col] - strlen(field->name)); + int left = padding / 2; printf(" %*.s%s%*.s |", left, " ", field->name, padding - left, " "); rowWidth += width[col] + 3; } @@ -834,16 +827,15 @@ static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) { putchar('\n'); } - -static int horizontalPrintResult(TAOS_RES* tres) { +static int horizontalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int precision = taos_result_precision(tres); int width[TSDB_MAX_COLUMNS]; for (int col = 0; col < num_fields; col++) { @@ -854,7 +846,7 @@ static int horizontalPrintResult(TAOS_RES* tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /* && !tscIsQueryWithLimit(tres)*/) { resShowMaxNum = DEFAULT_RES_SHOW_NUM; } @@ -862,34 +854,33 @@ static int horizontalPrintResult(TAOS_RES* tres) { int showMore = 1; do { - int32_t* length = taos_fetch_lengths(tres); + int32_t *length = taos_fetch_lengths(tres); if (numOfRows < resShowMaxNum) { for (int i = 0; i < num_fields; i++) { putchar(' '); - printField((const char*)row[i], fields + i, width[i], length[i], precision); + printField((const char *)row[i], fields + i, width[i], length[i], precision); putchar(' '); putchar('|'); } putchar('\n'); } else if (showMore) { - printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); - printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); - showMore = 0; + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); + showMore = 0; } numOfRows++; row = taos_fetch_row(tres); - } while(row != NULL); + } while (row != NULL); return numOfRows; } - int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { int numOfRows = 0; if (fname != NULL) { numOfRows = dumpResultToFile(fname, tres); - } else if(vertical) { + } else if (vertical) { numOfRows = verticalPrintResult(tres); } else { numOfRows = horizontalPrintResult(tres); @@ -899,13 +890,12 @@ int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { return numOfRows; } - void read_history() { // Initialize history memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE); history.hstart = 0; history.hend = 0; - char * line = NULL; + char *line = NULL; size_t line_size = 0; int read_size = 0; @@ -975,9 +965,9 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = calloc(1, tsMaxSQLStringLen+1); + char *cmd = calloc(1, tsMaxSQLStringLen + 1); size_t cmd_len = 0; - char * line = NULL; + char *line = NULL; size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { @@ -1087,5 +1077,5 @@ void shellGetGrantInfo(void *con) { } fprintf(stdout, "\n"); - #endif +#endif } diff --git a/src/kit/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c similarity index 99% rename from src/kit/shell/src/shellLinux.c rename to tools/shell/src/shellLinux.c index 93783b2055..81d8c0a35b 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -19,7 +19,11 @@ #include "shell.h" #include "shellCommand.h" #include "tkey.h" -#include "tulog.h" +#include "ulog.h" + +#include +#include +#include #define OPT_ABORT 1 /* �Cabort */ @@ -68,7 +72,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'P': if (arg) { - tsDnodeShellPort = atoi(arg); arguments->port = atoi(arg); } else { fprintf(stderr, "Invalid port\n"); @@ -216,7 +219,9 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { argp_parse(&argp, argc, argv, 0, 0, arguments); if (arguments->abort) { #ifndef _ALPINE + #if 0 error(10, 0, "ABORTED"); + #endif #else abort(); #endif diff --git a/src/kit/shell/src/shellMain.c b/tools/shell/src/shellMain.c similarity index 78% rename from src/kit/shell/src/shellMain.c rename to tools/shell/src/shellMain.c index 29713686ea..607dc79257 100644 --- a/src/kit/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -16,7 +16,7 @@ #include "os.h" #include "shell.h" #include "tconfig.h" -#include "tnettest.h" +#include "tglobal.h" pthread_t pid; static tsem_t cancelSem; @@ -28,23 +28,27 @@ void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { void *cancelHandler(void *arg) { setThreadName("cancelHandler"); - while(1) { + while (1) { if (tsem_wait(&cancelSem) != 0) { taosMsleep(10); continue; } #ifdef LINUX +#if 0 int64_t rid = atomic_val_compare_exchange_64(&result, result, 0); SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); taos_stop_query(pSql); taosReleaseRef(tscObjRef, rid); +#endif #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); #endif + printf("\nReceive ctrl+c or other signal, quit shell.\n"); + exit(0); } - + return NULL; } @@ -69,31 +73,29 @@ int checkVersion() { } // Global configurations -SShellArguments args = { - .host = NULL, +SShellArguments args = {.host = NULL, #ifndef TD_WINDOWS - .password = NULL, + .password = NULL, #endif - .user = NULL, - .database = NULL, - .timezone = NULL, - .is_raw_time = false, - .is_use_passwd = false, - .dump_config = false, - .file = "\0", - .dir = "\0", - .threadNum = 5, - .commands = NULL, - .pktLen = 1000, - .pktNum = 100, - .pktType = "TCP", - .netTestRole = NULL -}; + .user = NULL, + .database = NULL, + .timezone = NULL, + .is_raw_time = false, + .is_use_passwd = false, + .dump_config = false, + .file = "\0", + .dir = "\0", + .threadNum = 5, + .commands = NULL, + .pktLen = 1000, + .pktNum = 100, + .pktType = "TCP", + .netTestRole = NULL}; /* * Main function. */ -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { /*setlocale(LC_ALL, "en_US.UTF-8"); */ if (!checkVersion()) { @@ -102,6 +104,7 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); +#if 0 if (args.dump_config) { taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -123,9 +126,10 @@ int main(int argc, char* argv[]) { taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); exit(0); } +#endif /* Initialize the shell */ - TAOS* con = shellInit(&args); + TAOS *con = shellInit(&args); if (con == NULL) { exit(EXIT_FAILURE); }