diff --git a/2.0/src/query/tests/astTest.cpp b/2.0/src/query/tests/astTest.cpp index b7df749983..1109d25f29 100644 --- a/2.0/src/query/tests/astTest.cpp +++ b/2.0/src/query/tests/astTest.cpp @@ -632,4 +632,6 @@ void exprSerializeTest2() { TEST(testCase, astTest) { // exprSerializeTest2(); } -#endif \ No newline at end of file +#endif + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/cSortTest.cpp b/2.0/src/query/tests/cSortTest.cpp index aa5aa89afc..64e3b41b49 100644 --- a/2.0/src/query/tests/cSortTest.cpp +++ b/2.0/src/query/tests/cSortTest.cpp @@ -5,6 +5,7 @@ #include "tsdb.h" #include "qExtbuffer.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -121,4 +122,6 @@ TEST(testCase, columnsort_test) { printf("\n"); destroyColumnModel(pModel); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/histogramTest.cpp b/2.0/src/query/tests/histogramTest.cpp index 0266ecffc1..ff5ac08cdc 100644 --- a/2.0/src/query/tests/histogramTest.cpp +++ b/2.0/src/query/tests/histogramTest.cpp @@ -6,6 +6,7 @@ #include "taos.h" #include "qHistogram.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -140,3 +141,5 @@ TEST(testCase, heapsort) { // // free(pEntry); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/patternMatchTest.cpp b/2.0/src/query/tests/patternMatchTest.cpp index 091604c65c..3b46c91bc3 100644 --- a/2.0/src/query/tests/patternMatchTest.cpp +++ b/2.0/src/query/tests/patternMatchTest.cpp @@ -6,6 +6,7 @@ #include "qAggMain.h" #include "tcompare.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -84,3 +85,5 @@ TEST(testCase, patternMatchTest) { ret = patternMatch("%9", str, 2, &info); EXPECT_EQ(ret, TSDB_PATTERN_MATCH); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/percentileTest.cpp b/2.0/src/query/tests/percentileTest.cpp index 1b6951201a..7c4d85b3ae 100644 --- a/2.0/src/query/tests/percentileTest.cpp +++ b/2.0/src/query/tests/percentileTest.cpp @@ -7,6 +7,7 @@ #include "qPercentile.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -255,3 +256,5 @@ TEST(testCase, percentileTest) { unsignedDataTest(); largeDataTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/rangeMergeTest.cpp b/2.0/src/query/tests/rangeMergeTest.cpp index f7fc558ccf..9bbf47e56d 100644 --- a/2.0/src/query/tests/rangeMergeTest.cpp +++ b/2.0/src/query/tests/rangeMergeTest.cpp @@ -7,6 +7,7 @@ #include "qFilter.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -365,3 +366,5 @@ TEST(testCase, rangeMergeTest) { intDataTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp index 54ac0bf4e5..9724b98f7c 100644 --- a/2.0/src/query/tests/resultBufferTest.cpp +++ b/2.0/src/query/tests/resultBufferTest.cpp @@ -6,6 +6,7 @@ #include "taos.h" #include "tsdb.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -161,3 +162,5 @@ TEST(testCase, resultBufferTest) { writeDownTest(); recyclePageTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/tsBufTest.cpp b/2.0/src/query/tests/tsBufTest.cpp index 04c5a15252..62ffa785c6 100644 --- a/2.0/src/query/tests/tsBufTest.cpp +++ b/2.0/src/query/tests/tsBufTest.cpp @@ -9,6 +9,7 @@ #include "ttoken.h" #include "tutil.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wunused-but-set-variable" @@ -513,3 +514,5 @@ TEST(testCase, tsBufTest) { mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/2.0/src/query/tests/unitTest.cpp b/2.0/src/query/tests/unitTest.cpp index 1ed4cde406..57f66ef466 100644 --- a/2.0/src/query/tests/unitTest.cpp +++ b/2.0/src/query/tests/unitTest.cpp @@ -4,6 +4,7 @@ #include "taos.h" #include "tsdb.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" @@ -910,3 +911,4 @@ TEST(testCase, getTempFilePath_test) { printf("%s\n", path); } +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 99e48006b1..0383cc8aed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,8 +10,8 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) diff --git a/contrib/test/traft/CMakeLists.txt b/contrib/test/traft/CMakeLists.txt index d165c6d4dc..32e391646a 100644 --- a/contrib/test/traft/CMakeLists.txt +++ b/contrib/test/traft/CMakeLists.txt @@ -1,3 +1,2 @@ -add_subdirectory(rebalance_leader) -add_subdirectory(make_cluster) +add_subdirectory(single_node) diff --git a/contrib/test/traft/cluster/Makefile.2 b/contrib/test/traft/cluster/Makefile.2 new file mode 100644 index 0000000000..0a4f6ff325 --- /dev/null +++ b/contrib/test/traft/cluster/Makefile.2 @@ -0,0 +1,16 @@ +all: + gcc node10000.c -I ../../include/ ../../.libs/libraft.a -o node10000 -luv -llz4 -lpthread -g + gcc node10001.c -I ../../include/ ../../.libs/libraft.a -o node10001 -luv -llz4 -lpthread -g + gcc node10002.c -I ../../include/ ../../.libs/libraft.a -o node10002 -luv -llz4 -lpthread -g + gcc node10000_restart.c -I ../../include/ ../../.libs/libraft.a -o node10000_restart -luv -llz4 -lpthread -g + gcc node10001_restart.c -I ../../include/ ../../.libs/libraft.a -o node10001_restart -luv -llz4 -lpthread -g + gcc node10002_restart.c -I ../../include/ ../../.libs/libraft.a -o node10002_restart -luv -llz4 -lpthread -g +clean: + rm -f node10000 + rm -f node10001 + rm -f node10002 + rm -f node10000_restart + rm -f node10001_restart + rm -f node10002_restart + sh clear.sh + diff --git a/contrib/test/traft/make_cluster/clear.sh b/contrib/test/traft/cluster/clear.sh similarity index 100% rename from contrib/test/traft/make_cluster/clear.sh rename to contrib/test/traft/cluster/clear.sh diff --git a/contrib/test/traft/cluster/node10000.c b/contrib/test/traft/cluster/node10000.c new file mode 100644 index 0000000000..96e0b067c2 --- /dev/null +++ b/contrib/test/traft/cluster/node10000.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // array gives the peer replica infomation. + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10001"); + snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10002"); + uint32_t peersCount = 2; + r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10000_restart.c b/contrib/test/traft/cluster/node10000_restart.c new file mode 100644 index 0000000000..c9538552ed --- /dev/null +++ b/contrib/test/traft/cluster/node10000_restart.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // here only add self. + // peer replica information will restore from wal. + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10001.c b/contrib/test/traft/cluster/node10001.c new file mode 100644 index 0000000000..08636637ac --- /dev/null +++ b/contrib/test/traft/cluster/node10001.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // array gives the peer replica infomation. + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10000"); + snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10002"); + uint32_t peersCount = 2; + r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10001_restart.c b/contrib/test/traft/cluster/node10001_restart.c new file mode 100644 index 0000000000..10d64d76ef --- /dev/null +++ b/contrib/test/traft/cluster/node10001_restart.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // here only add self. + // peer replica information will restore from wal. + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10002.c b/contrib/test/traft/cluster/node10002.c new file mode 100644 index 0000000000..ebc3598075 --- /dev/null +++ b/contrib/test/traft/cluster/node10002.c @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // array gives the peer replica infomation. + char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; + memset(peers, 0, sizeof(peers)); + snprintf(peers[0], ADDRESS_LEN, "%s", "127.0.0.1:10000"); + snprintf(peers[1], ADDRESS_LEN, "%s", "127.0.0.1:10001"); + uint32_t peersCount = 2; + r = addRaftVoter(&raftEnv, peers, peersCount, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/cluster/node10002_restart.c b/contrib/test/traft/cluster/node10002_restart.c new file mode 100644 index 0000000000..d772e97b2b --- /dev/null +++ b/contrib/test/traft/cluster/node10002_restart.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has 3 replica. + // here only add self. + // peer replica information will restore from wal. + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/join_into_vgroup/Makefile.2 b/contrib/test/traft/join_into_vgroup/Makefile.2 new file mode 100644 index 0000000000..92f5b7269a --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/Makefile.2 @@ -0,0 +1,10 @@ +all: + gcc node_follower10000.c -I ../../include/ ../../.libs/libraft.a -o node_follower10000 -luv -llz4 -lpthread -g + gcc node_follower10001.c -I ../../include/ ../../.libs/libraft.a -o node_follower10001 -luv -llz4 -lpthread -g + gcc node_leader10002.c -I ../../include/ ../../.libs/libraft.a -o node_leader10002 -luv -llz4 -lpthread -g +clean: + rm -f node_follower10000 + rm -f node_follower10001 + rm -f node_leader10002 + sh clear.sh + diff --git a/contrib/test/traft/rebalance_leader/clear.sh b/contrib/test/traft/join_into_vgroup/clear.sh similarity index 100% rename from contrib/test/traft/rebalance_leader/clear.sh rename to contrib/test/traft/join_into_vgroup/clear.sh diff --git a/contrib/test/traft/join_into_vgroup/node_follower10000.c b/contrib/test/traft/join_into_vgroup/node_follower10000.c new file mode 100644 index 0000000000..b684ef199c --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/node_follower10000.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // add one replica + r = addRaftSpare(&raftEnv, 100, pFsm); + assert(r == 0); + + // for test: submit value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/join_into_vgroup/node_follower10001.c b/contrib/test/traft/join_into_vgroup/node_follower10001.c new file mode 100644 index 0000000000..27acd33917 --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/node_follower10001.c @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10001, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // add one replica + r = addRaftSpare(&raftEnv, 100, pFsm); + assert(r == 0); + + // for test: submit value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/join_into_vgroup/node_leader10002.c b/contrib/test/traft/join_into_vgroup/node_leader10002.c new file mode 100644 index 0000000000..b57af7777b --- /dev/null +++ b/contrib/test/traft/join_into_vgroup/node_leader10002.c @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include +#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +void joinRaftPeerCb(struct raft_change *req, int status) { + struct raft *r = req->data; + if (status != 0) { + fprintf(stderr, "joinRaftPeerCb error: %s \n", raft_errmsg(r)); + } else { + fprintf(stderr, "joinRaftPeerCb ok \n"); + } + raft_free(req); +} + +int main(int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 10002, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // add one replica + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + printRaftState(getRaft(&raftEnv, 100)); + + // wait for being leader + while (1) { + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + if (r->state == RAFT_LEADER) { + break; + } + } + + // join peers + r = joinRaftPeer(&raftEnv, 100, "127.0.0.1", 10000, joinRaftPeerCb); + assert(r == 0); + + // wait for join over + sleep(2); + + r = joinRaftPeer(&raftEnv, 100, "127.0.0.1", 10001, joinRaftPeerCb); + assert(r == 0); + + // for test: submit value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/contrib/test/traft/make_cluster/CMakeLists.txt b/contrib/test/traft/make_cluster/CMakeLists.txt deleted file mode 100644 index afd19d5435..0000000000 --- a/contrib/test/traft/make_cluster/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_executable(makeCluster "") -target_sources(makeCluster - PRIVATE - "raftMain.c" - "raftServer.c" - "config.c" - "console.c" - "simpleHash.c" - "util.c" -) -target_link_libraries(makeCluster PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/make_cluster/common.h b/contrib/test/traft/make_cluster/common.h deleted file mode 100644 index df7422033a..0000000000 --- a/contrib/test/traft/make_cluster/common.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef TRAFT_COMMON_H -#define TRAFT_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#define COMMAND_LEN 512 -#define MAX_CMD_COUNT 10 -#define TOKEN_LEN 128 -#define MAX_PEERS_COUNT 19 - -#define HOST_LEN 64 -#define ADDRESS_LEN (HOST_LEN * 2) -#define BASE_DIR_LEN 128 - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/make_cluster/config.c b/contrib/test/traft/make_cluster/config.c deleted file mode 100644 index 3b96839fd9..0000000000 --- a/contrib/test/traft/make_cluster/config.c +++ /dev/null @@ -1,64 +0,0 @@ -#include "config.h" -#include -#include -#include - -void addrToString(const char *host, uint16_t port, char *addr, int len) { snprintf(addr, len, "%s:%hu", host, port); } - -void parseAddr(const char *addr, char *host, int len, uint16_t *port) { - char *tmp = (char *)malloc(strlen(addr) + 1); - strcpy(tmp, addr); - - char *context; - char *separator = ":"; - char *token = strtok_r(tmp, separator, &context); - if (token) { - snprintf(host, len, "%s", token); - } - - token = strtok_r(NULL, separator, &context); - if (token) { - sscanf(token, "%hu", port); - } - - free(tmp); -} - -int parseConf(int argc, char **argv, RaftServerConfig *pConf) { - memset(pConf, 0, sizeof(*pConf)); - - int option_index, option_value; - option_index = 0; - static struct option long_options[] = {{"help", no_argument, NULL, 'h'}, - {"addr", required_argument, NULL, 'a'}, - {"dir", required_argument, NULL, 'd'}, - {NULL, 0, NULL, 0}}; - - while ((option_value = getopt_long(argc, argv, "ha:d:", long_options, &option_index)) != -1) { - switch (option_value) { - case 'a': { - parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); - break; - } - - case 'd': { - snprintf(pConf->baseDir, sizeof(pConf->baseDir), "%s", optarg); - break; - } - - case 'h': { - return -2; - } - - default: { return -2; } - } - } - - return 0; -} - -void printConf(RaftServerConfig *pConf) { - printf("\n---printConf: \n"); - printf("me: [%s:%hu] \n", pConf->me.host, pConf->me.port); - printf("dataDir: [%s] \n\n", pConf->baseDir); -} diff --git a/contrib/test/traft/make_cluster/config.h b/contrib/test/traft/make_cluster/config.h deleted file mode 100644 index 13c43d0d28..0000000000 --- a/contrib/test/traft/make_cluster/config.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef TRAFT_CONFIG_H -#define TRAFT_CONFIG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "common.h" - -typedef struct { - char host[HOST_LEN]; - uint16_t port; -} Addr; - -typedef struct { - Addr me; - char baseDir[BASE_DIR_LEN]; -} RaftServerConfig; - -void addrToString(const char *host, uint16_t port, char *addr, int len); -void parseAddr(const char *addr, char *host, int len, uint16_t *port); -int parseConf(int argc, char **argv, RaftServerConfig *pConf); -void printConf(RaftServerConfig *pConf); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/make_cluster/console.c b/contrib/test/traft/make_cluster/console.c deleted file mode 100644 index b00550c681..0000000000 --- a/contrib/test/traft/make_cluster/console.c +++ /dev/null @@ -1,202 +0,0 @@ -#include "console.h" -#include -#include -#include -#include "raftServer.h" -#include "util.h" - -void printHelp() { - printf("---------------------\n"); - printf("help: \n\n"); - printf("create a vgroup with 3 replicas: \n"); - printf("create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 \n"); - printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10002 \n"); - printf("create vnode voter vid 100 peers 127.0.0.1:10000 127.0.0.1:10001 \n"); - printf("\n"); - printf("create a vgroup with only one replica: \n"); - printf("create vnode voter vid 200 \n"); - printf("\n"); - printf("add vnode into vgroup: \n"); - printf("create vnode spare vid 100 ---- run at 127.0.0.1:10003\n"); - printf("join vnode vid 100 addr 127.0.0.1:10003 ---- run at leader of vgroup 100\n"); - printf("\n"); - printf("run \n"); - printf("put 0 key value \n"); - printf("get 0 key \n"); - printf("---------------------\n"); -} - -void console(RaftServer *pRaftServer) { - while (1) { - int ret; - char cmdBuf[COMMAND_LEN]; - memset(cmdBuf, 0, sizeof(cmdBuf)); - printf("(console)> "); - char *retp = fgets(cmdBuf, COMMAND_LEN, stdin); - if (!retp) { - exit(-1); - } - - int pos = strlen(cmdBuf); - if (cmdBuf[pos - 1] == '\n') { - cmdBuf[pos - 1] = '\0'; - } - - if (strncmp(cmdBuf, "", COMMAND_LEN) == 0) { - continue; - } - - char cmds[MAX_CMD_COUNT][TOKEN_LEN]; - memset(cmds, 0, sizeof(cmds)); - - int cmdCount; - cmdCount = splitString(cmdBuf, " ", cmds, MAX_CMD_COUNT); - - if (strcmp(cmds[0], "create") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[3], "vid") == 0) { - uint16_t vid; - sscanf(cmds[4], "%hu", &vid); - - if (strcmp(cmds[2], "voter") == 0) { - char peers[MAX_PEERS_COUNT][ADDRESS_LEN]; - memset(peers, 0, sizeof(peers)); - uint32_t peersCount = 0; - - if (strcmp(cmds[5], "peers") == 0 && cmdCount > 6) { - // create vnode voter vid 100 peers 127.0.0.1:10001 127.0.0.1:10002 - for (int i = 6; i < cmdCount; ++i) { - snprintf(peers[i - 6], ADDRESS_LEN, "%s", cmds[i]); - peersCount++; - } - } else { - // create vnode voter vid 200 - } - ret = addRaftVoter(pRaftServer, peers, peersCount, vid); - if (ret == 0) { - printf("create vnode voter ok \n"); - } else { - printf("create vnode voter error \n"); - } - } else if (strcmp(cmds[2], "spare") == 0) { - ret = addRaftSpare(pRaftServer, vid); - if (ret == 0) { - printf("create vnode spare ok \n"); - } else { - printf("create vnode spare error \n"); - } - } else { - printHelp(); - } - - } else if (strcmp(cmds[0], "join") == 0 && strcmp(cmds[1], "vnode") == 0 && strcmp(cmds[2], "vid") == 0 && - strcmp(cmds[4], "addr") == 0 && cmdCount == 6) { - // join vnode vid 100 addr 127.0.0.1:10004 - - char * address = cmds[5]; - char host[64]; - uint16_t port; - parseAddr(address, host, sizeof(host), &port); - - uint16_t vid; - sscanf(cmds[3], "%hu", &vid); - - HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); - if (*pp == NULL) { - printf("vid:%hu not found \n", vid); - break; - } - RaftInstance *pRaftInstance = (*pp)->data; - - uint64_t destRaftId = encodeRaftId(host, port, vid); - - struct raft_change *req = raft_malloc(sizeof(*req)); - RaftJoin * pRaftJoin = raft_malloc(sizeof(*pRaftJoin)); - pRaftJoin->r = &pRaftInstance->raft; - pRaftJoin->joinId = destRaftId; - req->data = pRaftJoin; - ret = raft_add(&pRaftInstance->raft, req, destRaftId, address, raftChangeAddCb); - if (ret != 0) { - printf("raft_add error: %s \n", raft_errmsg(&pRaftInstance->raft)); - } - - } else if (strcmp(cmds[0], "dropnode") == 0) { - } else if (strcmp(cmds[0], "state") == 0) { - pRaftServer->raftInstances.print(&pRaftServer->raftInstances); - for (size_t i = 0; i < pRaftServer->raftInstances.length; ++i) { - HashNode *ptr = pRaftServer->raftInstances.table[i]; - if (ptr != NULL) { - while (ptr != NULL) { - RaftInstance *pRaftInstance = ptr->data; - printf("instance vid:%hu raftId:%llu \n", ptr->vgroupId, pRaftInstance->raftId); - printRaftState(&pRaftInstance->raft); - printf("\n"); - ptr = ptr->next; - } - printf("\n"); - } - } - - } else if (strcmp(cmds[0], "put") == 0 && cmdCount == 4) { - uint16_t vid; - sscanf(cmds[1], "%hu", &vid); - char * key = cmds[2]; - char * value = cmds[3]; - HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); - if (*pp == NULL) { - printf("vid:%hu not found \n", vid); - break; - } - RaftInstance *pRaftInstance = (*pp)->data; - - char *raftValue = malloc(TOKEN_LEN * 2 + 3); - snprintf(raftValue, TOKEN_LEN * 2 + 3, "%s--%s", key, value); - putValue(&pRaftInstance->raft, raftValue); - free(raftValue); - - } else if (strcmp(cmds[0], "run") == 0) { - pthread_t tidRaftServer; - pthread_create(&tidRaftServer, NULL, startServerFunc, pRaftServer); - - } else if (strcmp(cmds[0], "get") == 0 && cmdCount == 3) { - uint16_t vid; - sscanf(cmds[1], "%hu", &vid); - char * key = cmds[2]; - HashNode **pp = pRaftServer->raftInstances.find(&pRaftServer->raftInstances, vid); - if (*pp == NULL) { - printf("vid:%hu not found \n", vid); - break; - } - RaftInstance * pRaftInstance = (*pp)->data; - SimpleHash * pKV = pRaftInstance->fsm.data; - SimpleHashNode **ppNode = pKV->find_cstr(pKV, key); - if (*ppNode == NULL) { - printf("key:%s not found \n", key); - } else { - printf("find key:%s value:%s \n", key, (char *)((*ppNode)->data)); - } - - } else if (strcmp(cmds[0], "transfer") == 0) { - } else if (strcmp(cmds[0], "state") == 0) { - } else if (strcmp(cmds[0], "snapshot") == 0) { - } else if (strcmp(cmds[0], "exit") == 0) { - exit(0); - - } else if (strcmp(cmds[0], "quit") == 0) { - exit(0); - - } else if (strcmp(cmds[0], "help") == 0) { - printHelp(); - - } else { - printf("unknown command: %s \n", cmdBuf); - printHelp(); - } - - /* - printf("cmdBuf: [%s] \n", cmdBuf); - printf("cmdCount : %d \n", cmdCount); - for (int i = 0; i < MAX_CMD_COUNT; ++i) { - printf("cmd%d : %s \n", i, cmds[i]); - } - */ - } -} diff --git a/contrib/test/traft/make_cluster/console.h b/contrib/test/traft/make_cluster/console.h deleted file mode 100644 index f9ed12baf5..0000000000 --- a/contrib/test/traft/make_cluster/console.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef TRAFT_CONSOLE_H -#define TRAFT_CONSOLE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include "common.h" -#include "raftServer.h" - -void console(RaftServer *pRaftServer); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/make_cluster/raftMain.c b/contrib/test/traft/make_cluster/raftMain.c deleted file mode 100644 index e25636de91..0000000000 --- a/contrib/test/traft/make_cluster/raftMain.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" -#include "config.h" -#include "console.h" -#include "raftServer.h" -#include "simpleHash.h" -#include "util.h" - -const char *exe_name; - -void *startConsoleFunc(void *param) { - RaftServer *pRaftServer = (RaftServer *)param; - console(pRaftServer); - return NULL; -} - -void usage() { - printf("\nusage: \n"); - printf("%s --addr=127.0.0.1:10000 --dir=./data \n", exe_name); - printf("\n"); -} - -RaftServerConfig gConfig; -RaftServer gRaftServer; - -int main(int argc, char **argv) { - srand(time(NULL)); - int32_t ret; - - exe_name = argv[0]; - if (argc < 3) { - usage(); - exit(-1); - } - - ret = parseConf(argc, argv, &gConfig); - if (ret != 0) { - usage(); - exit(-1); - } - printConf(&gConfig); - - if (!dirOK(gConfig.baseDir)) { - ret = mkdir(gConfig.baseDir, 0775); - if (ret != 0) { - fprintf(stderr, "mkdir error, %s \n", gConfig.baseDir); - exit(-1); - } - } - - ret = raftServerInit(&gRaftServer, &gConfig); - if (ret != 0) { - fprintf(stderr, "raftServerInit error \n"); - exit(-1); - } - - /* - pthread_t tidRaftServer; - pthread_create(&tidRaftServer, NULL, startServerFunc, &gRaftServer); - */ - - pthread_t tidConsole; - pthread_create(&tidConsole, NULL, startConsoleFunc, &gRaftServer); - - while (1) { - sleep(10); - } - - return 0; -} diff --git a/contrib/test/traft/make_cluster/raftServer.c b/contrib/test/traft/make_cluster/raftServer.c deleted file mode 100644 index bbf67b9420..0000000000 --- a/contrib/test/traft/make_cluster/raftServer.c +++ /dev/null @@ -1,286 +0,0 @@ -#include "raftServer.h" -#include -#include -#include "common.h" -#include "simpleHash.h" -#include "util.h" - -void *startServerFunc(void *param) { - RaftServer *pRaftServer = (RaftServer *)param; - int32_t r = raftServerStart(pRaftServer); - assert(r == 0); - return NULL; -} - -void raftChangeAssignCb(struct raft_change *req, int status) { - struct raft *r = req->data; - if (status != 0) { - printf("raftChangeAssignCb error: %s \n", raft_errmsg(r)); - } else { - printf("raftChangeAssignCb ok \n"); - } - raft_free(req); -} - -void raftChangeAddCb(struct raft_change *req, int status) { - RaftJoin *pRaftJoin = req->data; - if (status != 0) { - printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r)); - } else { - struct raft_change *req2 = raft_malloc(sizeof(*req2)); - req2->data = pRaftJoin->r; - int ret = raft_assign(pRaftJoin->r, req2, pRaftJoin->joinId, RAFT_VOTER, raftChangeAssignCb); - if (ret != 0) { - printf("raftChangeAddCb error: %s \n", raft_errmsg(pRaftJoin->r)); - } - } - raft_free(req->data); - raft_free(req); -} - -int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { - // get fsm data - SimpleHash *sh = pFsm->data; - - // get commit value - char *msg = (char *)buf->base; - printf("fsm apply: [%s] \n", msg); - char arr[2][TOKEN_LEN]; - int r = splitString(msg, "--", arr, 2); - assert(r == 2); - - // do the value on fsm - sh->insert_cstr(sh, arr[0], arr[1]); - - raft_free(buf->base); - return 0; -} - -void putValueCb(struct raft_apply *req, int status, void *result) { - struct raft *r = req->data; - if (status != 0) { - printf("putValueCb error: %s \n", raft_errmsg(r)); - } else { - printf("putValueCb: %s \n", "ok"); - } - raft_free(req); -} - -void putValue(struct raft *r, const char *value) { - struct raft_buffer buf; - - buf.len = strlen(value) + 1; - buf.base = raft_malloc(buf.len); - snprintf(buf.base, buf.len, "%s", value); - - struct raft_apply *req = raft_malloc(sizeof(*req)); - req->data = r; - int ret = raft_apply(r, req, &buf, 1, putValueCb); - if (ret == 0) { - printf("put %s \n", (char *)buf.base); - } else { - printf("put error: %s \n", raft_errmsg(r)); - } -} - -const char *state2String(unsigned short state) { - if (state == RAFT_UNAVAILABLE) { - return "RAFT_UNAVAILABLE"; - - } else if (state == RAFT_FOLLOWER) { - return "RAFT_FOLLOWER"; - - } else if (state == RAFT_CANDIDATE) { - return "RAFT_CANDIDATE"; - - } else if (state == RAFT_LEADER) { - return "RAFT_LEADER"; - } - return "UNKNOWN_RAFT_STATE"; -} - -void printRaftConfiguration(struct raft_configuration *c) { - printf("configuration: \n"); - for (int i = 0; i < c->n; ++i) { - printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); - } -} - -void printRaftState(struct raft *r) { - printf("----Raft State: -----------\n"); - printf("mem_addr: %p \n", r); - printf("my_id: %llu \n", r->id); - printf("address: %s \n", r->address); - printf("current_term: %llu \n", r->current_term); - printf("voted_for: %llu \n", r->voted_for); - printf("role: %s \n", state2String(r->state)); - printf("commit_index: %llu \n", r->commit_index); - printf("last_applied: %llu \n", r->last_applied); - printf("last_stored: %llu \n", r->last_stored); - - printf("configuration_index: %llu \n", r->configuration_index); - printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); - printRaftConfiguration(&r->configuration); - - printf("----------------------------\n"); -} - -int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid) { - int ret; - - RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance)); - assert(pRaftInstance != NULL); - - // init raftId - pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); - - // init dir - snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir, - pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId); - - if (!dirOK(pRaftInstance->dir)) { - ret = mkdir(pRaftInstance->dir, 0775); - if (ret != 0) { - fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir); - assert(0); - } - } - - // init fsm - pRaftInstance->fsm.data = newSimpleHash(2); - pRaftInstance->fsm.apply = fsmApplyCb; - - // init io - ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport); - if (ret != 0) { - fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft - ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId, - pRaftServer->address); - if (ret != 0) { - fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft_configuration - struct raft_configuration conf; - raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_VOTER); - for (int i = 0; i < peersCount; ++i) { - char * peerAddress = peers[i]; - char host[64]; - uint16_t port; - parseAddr(peerAddress, host, sizeof(host), &port); - uint64_t raftId = encodeRaftId(host, port, vid); - raft_configuration_add(&conf, raftId, peers[i], RAFT_VOTER); - } - raft_bootstrap(&pRaftInstance->raft, &conf); - - // start raft - ret = raft_start(&pRaftInstance->raft); - if (ret != 0) { - fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // add raft instance into raft server - pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance); - - return 0; -} - -int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid) { - int ret; - - RaftInstance *pRaftInstance = malloc(sizeof(*pRaftInstance)); - assert(pRaftInstance != NULL); - - // init raftId - pRaftInstance->raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); - - // init dir - snprintf(pRaftInstance->dir, sizeof(pRaftInstance->dir), "%s/%s_%hu_%hu_%llu", pRaftServer->baseDir, - pRaftServer->host, pRaftServer->port, vid, pRaftInstance->raftId); - ret = mkdir(pRaftInstance->dir, 0775); - if (ret != 0) { - fprintf(stderr, "mkdir error, %s \n", pRaftInstance->dir); - assert(0); - } - - // init fsm - pRaftInstance->fsm.data = newSimpleHash(2); - pRaftInstance->fsm.apply = fsmApplyCb; - - // init io - ret = raft_uv_init(&pRaftInstance->io, &pRaftServer->loop, pRaftInstance->dir, &pRaftServer->transport); - if (ret != 0) { - fprintf(stderr, "raft_uv_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft - ret = raft_init(&pRaftInstance->raft, &pRaftInstance->io, &pRaftInstance->fsm, pRaftInstance->raftId, - pRaftServer->address); - if (ret != 0) { - fprintf(stderr, "raft_init error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // init raft_configuration - struct raft_configuration conf; - raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftInstance->raftId, pRaftServer->address, RAFT_SPARE); - raft_bootstrap(&pRaftInstance->raft, &conf); - - // start raft - ret = raft_start(&pRaftInstance->raft); - if (ret != 0) { - fprintf(stderr, "raft_start error, %s \n", raft_errmsg(&pRaftInstance->raft)); - assert(0); - } - - // add raft instance into raft server - pRaftServer->raftInstances.insert(&pRaftServer->raftInstances, vid, pRaftInstance); - - return 0; -} - -int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf) { - int ret; - - // init host, port, address, dir - snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); - pRaftServer->port = pConf->me.port; - snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); - snprintf(pRaftServer->baseDir, sizeof(pRaftServer->baseDir), "%s", pConf->baseDir); - - // init loop - ret = uv_loop_init(&pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); - assert(0); - } - - // init network - ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); - assert(0); - } - - // init raft instance container - initIdHash(&pRaftServer->raftInstances, 2); - - return 0; -} - -int32_t raftServerStart(RaftServer *pRaftServer) { - // start loop - uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); - return 0; -} - -void raftServerStop(RaftServer *pRaftServer) {} diff --git a/contrib/test/traft/make_cluster/raftServer.h b/contrib/test/traft/make_cluster/raftServer.h deleted file mode 100644 index b6dbddb2b7..0000000000 --- a/contrib/test/traft/make_cluster/raftServer.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef TDENGINE_RAFT_SERVER_H -#define TDENGINE_RAFT_SERVER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include "common.h" -#include "config.h" -#include "raft.h" -#include "raft/uv.h" -#include "simpleHash.h" - -typedef struct RaftJoin { - struct raft *r; - raft_id joinId; -} RaftJoin; - -typedef struct { - raft_id raftId; - char dir[BASE_DIR_LEN * 2]; - struct raft_fsm fsm; - struct raft_io io; - struct raft raft; -} RaftInstance; - -typedef struct { - char host[HOST_LEN]; - uint16_t port; - char address[ADDRESS_LEN]; /* Raft instance address */ - char baseDir[BASE_DIR_LEN]; /* Raft instance address */ - - struct uv_loop_s loop; /* UV loop */ - struct raft_uv_transport transport; /* UV I/O backend transport */ - - IdHash raftInstances; /* multi raft instances. traft use IdHash to manager multi vgroup inside, here we can use IdHash - too. */ -} RaftServer; - -void * startServerFunc(void *param); -int32_t addRaftVoter(RaftServer *pRaftServer, char peers[][ADDRESS_LEN], uint32_t peersCount, uint16_t vid); -int32_t addRaftSpare(RaftServer *pRaftServer, uint16_t vid); - -int32_t raftServerInit(RaftServer *pRaftServer, const RaftServerConfig *pConf); -int32_t raftServerStart(RaftServer *pRaftServer); -void raftServerStop(RaftServer *pRaftServer); - -int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result); -void putValueCb(struct raft_apply *req, int status, void *result); -void putValue(struct raft *r, const char *value); - -void raftChangeAddCb(struct raft_change *req, int status); - -const char *state2String(unsigned short state); -void printRaftConfiguration(struct raft_configuration *c); -void printRaftState(struct raft *r); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RAFT_SERVER_H diff --git a/contrib/test/traft/make_cluster/simpleHash.c b/contrib/test/traft/make_cluster/simpleHash.c deleted file mode 100644 index 6694843874..0000000000 --- a/contrib/test/traft/make_cluster/simpleHash.c +++ /dev/null @@ -1,218 +0,0 @@ -#include "simpleHash.h" - -uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed) { - // Similar to murmur hash - const uint32_t m = 0xc6a4a793; - const uint32_t r = 24; - const char* limit = data + n; - uint32_t h = seed ^ (n * m); - - // Pick up four bytes at a time - while (data + 4 <= limit) { - // uint32_t w = DecodeFixed32(data); - uint32_t w; - memcpy(&w, data, 4); - - data += 4; - h += w; - h *= m; - h ^= (h >> 16); - } - - // Pick up remaining bytes - switch (limit - data) { - case 3: - h += (unsigned char)(data[2]) << 16; - do { - } while (0); - case 2: - h += (unsigned char)(data[1]) << 8; - do { - } while (0); - case 1: - h += (unsigned char)(data[0]); - h *= m; - h ^= (h >> r); - break; - } - return h; -} - -int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data) { - return insertSimpleHash(ths, key, strlen(key) + 1, data, strlen(data) + 1); -} - -int removeCStrSimpleHash(struct SimpleHash* ths, char* key) { return removeSimpleHash(ths, key, strlen(key) + 1); } - -SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key) { - return findSimpleHash(ths, key, strlen(key) + 1); -} - -int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen) { - SimpleHashNode** pp = ths->find(ths, key, keyLen); - if (*pp != NULL) { - fprintf(stderr, "insertSimpleHash, already has key \n"); - return -1; - } - - SimpleHashNode* node = malloc(sizeof(*node)); - node->hashCode = ths->hashFunc(key, keyLen); - node->key = malloc(keyLen); - node->keyLen = keyLen; - memcpy(node->key, key, keyLen); - node->data = malloc(dataLen); - node->dataLen = dataLen; - memcpy(node->data, data, dataLen); - node->next = NULL; - - // printf("insertSimpleHash: <%s, %ld, %s, %ld, %u> \n", node->key, node->keyLen, node->data, node->dataLen, - // node->hashCode); - - size_t index = node->hashCode & (ths->length - 1); - - SimpleHashNode* ptr = ths->table[index]; - if (ptr != NULL) { - node->next = ptr; - ths->table[index] = node; - - } else { - ths->table[index] = node; - } - ths->elems++; - if (ths->elems > 2 * ths->length) { - ths->resize(ths); - } - - return 0; -} - -int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) { - SimpleHashNode** pp = ths->find(ths, key, keyLen); - if (*pp == NULL) { - fprintf(stderr, "removeSimpleHash, key not exist \n"); - return -1; - } - - SimpleHashNode* del = *pp; - *pp = del->next; - free(del->key); - free(del->data); - free(del); - ths->elems--; - - return 0; -} - -SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen) { - uint32_t hashCode = ths->hashFunc(key, keyLen); - // size_t index = hashCode % ths->length; - size_t index = hashCode & (ths->length - 1); - - // printf("findSimpleHash: %s %ld %u \n", key, keyLen, hashCode); - - SimpleHashNode** pp = &(ths->table[index]); - while (*pp != NULL && ((*pp)->hashCode != hashCode || memcmp(key, (*pp)->key, keyLen) != 0)) { - pp = &((*pp)->next); - } - - return pp; -} - -void printCStrSimpleHash(struct SimpleHash* ths) { - printf("\n--- printCStrSimpleHash: elems:%d length:%d \n", ths->elems, ths->length); - for (size_t i = 0; i < ths->length; ++i) { - SimpleHashNode* ptr = ths->table[i]; - if (ptr != NULL) { - printf("%zu: ", i); - while (ptr != NULL) { - printf("<%u, %s, %ld, %s, %ld> ", ptr->hashCode, (char*)ptr->key, ptr->keyLen, (char*)ptr->data, ptr->dataLen); - ptr = ptr->next; - } - printf("\n"); - } - } - printf("---------------\n"); -} - -void destroySimpleHash(struct SimpleHash* ths) { - for (size_t i = 0; i < ths->length; ++i) { - SimpleHashNode* ptr = ths->table[i]; - while (ptr != NULL) { - SimpleHashNode* tmp = ptr; - ptr = ptr->next; - free(tmp->key); - free(tmp->data); - free(tmp); - } - } - - ths->length = 0; - ths->elems = 0; - free(ths->table); - free(ths); -} - -void resizeSimpleHash(struct SimpleHash* ths) { - uint32_t new_length = ths->length; - while (new_length < ths->elems) { - new_length *= 2; - } - - printf("resizeSimpleHash: %p from %u to %u \n", ths, ths->length, new_length); - - SimpleHashNode** new_table = malloc(new_length * sizeof(SimpleHashNode*)); - memset(new_table, 0, new_length * sizeof(SimpleHashNode*)); - - uint32_t count = 0; - for (uint32_t i = 0; i < ths->length; i++) { - if (ths->table[i] == NULL) { - continue; - } - - SimpleHashNode* it = ths->table[i]; - while (it != NULL) { - SimpleHashNode* move_node = it; - it = it->next; - - // move move_node - move_node->next = NULL; - size_t index = move_node->hashCode & (new_length - 1); - - SimpleHashNode* ptr = new_table[index]; - if (ptr != NULL) { - move_node->next = ptr; - new_table[index] = move_node; - } else { - new_table[index] = move_node; - } - count++; - } - } - - assert(ths->elems == count); - free(ths->table); - ths->table = new_table; - ths->length = new_length; -} - -uint32_t simpleHashFunc(const char* key, size_t keyLen) { return mySimpleHash(key, keyLen, 1); } - -struct SimpleHash* newSimpleHash(size_t length) { - struct SimpleHash* ths = malloc(sizeof(*ths)); - - ths->length = length; - ths->elems = 0; - ths->table = malloc(length * sizeof(SimpleHashNode*)); - memset(ths->table, 0, length * sizeof(SimpleHashNode*)); - - ths->insert = insertSimpleHash; - ths->remove = removeSimpleHash; - ths->find = findSimpleHash; - ths->insert_cstr = insertCStrSimpleHash; - ths->remove_cstr = removeCStrSimpleHash; - ths->find_cstr = findCStrSimpleHash; - ths->print_cstr = printCStrSimpleHash; - ths->destroy = destroySimpleHash; - ths->resize = resizeSimpleHash; - ths->hashFunc = simpleHashFunc; -} diff --git a/contrib/test/traft/make_cluster/simpleHash.h b/contrib/test/traft/make_cluster/simpleHash.h deleted file mode 100644 index c6fcd93888..0000000000 --- a/contrib/test/traft/make_cluster/simpleHash.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef __SIMPLE_HASH_H__ -#define __SIMPLE_HASH_H__ - -#include -#include -#include -#include -#include - -uint32_t mySimpleHash(const char* data, size_t n, uint32_t seed); - -typedef struct SimpleHashNode { - uint32_t hashCode; - void* key; - size_t keyLen; - void* data; - size_t dataLen; - struct SimpleHashNode* next; -} SimpleHashNode; - -typedef struct SimpleHash { - // public: - - int (*insert)(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen); - int (*remove)(struct SimpleHash* ths, char* key, size_t keyLen); - SimpleHashNode** (*find)(struct SimpleHash* ths, char* key, size_t keyLen); - - // wrapper - int (*insert_cstr)(struct SimpleHash* ths, char* key, char* data); - int (*remove_cstr)(struct SimpleHash* ths, char* key); - SimpleHashNode** (*find_cstr)(struct SimpleHash* ths, char* key); - - void (*print_cstr)(struct SimpleHash* ths); - void (*destroy)(struct SimpleHash* ths); - - uint32_t length; - uint32_t elems; - - // private: - void (*resize)(struct SimpleHash* ths); - uint32_t (*hashFunc)(const char* key, size_t keyLen); - - SimpleHashNode** table; - -} SimpleHash; - -int insertCStrSimpleHash(struct SimpleHash* ths, char* key, char* data); -int removeCStrSimpleHash(struct SimpleHash* ths, char* key); -SimpleHashNode** findCStrSimpleHash(struct SimpleHash* ths, char* key); -void printCStrSimpleHash(struct SimpleHash* ths); - -int insertSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen, char* data, size_t dataLen); -int removeSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen); -SimpleHashNode** findSimpleHash(struct SimpleHash* ths, char* key, size_t keyLen); -void destroySimpleHash(struct SimpleHash* ths); -void resizeSimpleHash(struct SimpleHash* ths); -uint32_t simpleHashFunc(const char* key, size_t keyLen); - -struct SimpleHash* newSimpleHash(size_t length); - -#endif diff --git a/contrib/test/traft/make_cluster/util.c b/contrib/test/traft/make_cluster/util.c deleted file mode 100644 index ff704f3660..0000000000 --- a/contrib/test/traft/make_cluster/util.c +++ /dev/null @@ -1,45 +0,0 @@ -#include "util.h" -#include -#include -#include - -int dirOK(const char *path) { - DIR *dir = opendir(path); - if (dir != NULL) { - closedir(dir); - return 1; - } else { - return 0; - } -} - -int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr) { - if (n_arr <= 0) { - return -1; - } - - char *tmp = (char *)malloc(strlen(str) + 1); - strcpy(tmp, str); - char *context; - int n = 0; - - char *token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - strncpy(arr[n], token, TOKEN_LEN); - n++; - - while (1) { - token = strtok_r(NULL, separator, &context); - if (!token || n >= n_arr) { - goto ret; - } - strncpy(arr[n], token, TOKEN_LEN); - n++; - } - -ret: - free(tmp); - return n; -} diff --git a/contrib/test/traft/make_cluster/util.h b/contrib/test/traft/make_cluster/util.h deleted file mode 100644 index fb4ccb9c5c..0000000000 --- a/contrib/test/traft/make_cluster/util.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef TRAFT_UTIL_H -#define TRAFT_UTIL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "common.h" - -int dirOK(const char *path); -int splitString(const char *str, char *separator, char (*arr)[TOKEN_LEN], int n_arr); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/contrib/test/traft/rebalance_leader/CMakeLists.txt b/contrib/test/traft/rebalance_leader/CMakeLists.txt deleted file mode 100644 index 92640bdd80..0000000000 --- a/contrib/test/traft/rebalance_leader/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_executable(rebalanceLeader "") -target_sources(rebalanceLeader - PRIVATE - "raftMain.c" - "raftServer.c" -) -target_link_libraries(rebalanceLeader PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/rebalance_leader/common.h b/contrib/test/traft/rebalance_leader/common.h deleted file mode 100644 index 0229c29cf7..0000000000 --- a/contrib/test/traft/rebalance_leader/common.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef TDENGINE_COMMON_H -#define TDENGINE_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#define MAX_INSTANCE_NUM 100 - -#define MAX_PEERS 10 -#define COMMAND_LEN 1024 -#define TOKEN_LEN 128 -#define DIR_LEN 256 -#define HOST_LEN 64 -#define ADDRESS_LEN (HOST_LEN + 16) - -typedef struct { - char host[HOST_LEN]; - uint32_t port; -} Addr; - -typedef struct { - Addr me; - Addr peers[MAX_PEERS]; - int peersCount; - char dir[DIR_LEN]; - char dataDir[DIR_LEN + HOST_LEN * 2]; -} SRaftServerConfig; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_COMMON_H diff --git a/contrib/test/traft/rebalance_leader/raftMain.c b/contrib/test/traft/rebalance_leader/raftMain.c deleted file mode 100644 index 70dc191997..0000000000 --- a/contrib/test/traft/rebalance_leader/raftMain.c +++ /dev/null @@ -1,678 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "raftServer.h" -#include "common.h" - -const char *exe_name; - -typedef struct LeaderState { - char address[48]; - int leaderCount; - -} LeaderState; - -#define NODE_COUNT 3 -LeaderState leaderStates[NODE_COUNT]; - -void printLeaderCount() { - for (int i = 0; i < NODE_COUNT; ++i) { - printf("%s: leaderCount:%d \n", leaderStates[i].address, leaderStates[i].leaderCount); - } -} - -void updateLeaderStates(SRaftServer *pRaftServer) { - for (int i = 0; i < pRaftServer->instance[0].raft.configuration.n; ++i) { - snprintf(leaderStates[i].address, sizeof(leaderStates[i].address), "%s", pRaftServer->instance[0].raft.configuration.servers[i].address); - leaderStates[i].leaderCount = 0; - } - - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - struct raft *r = &pRaftServer->instance[i].raft; - - char leaderAddress[128]; - memset(leaderAddress, 0, sizeof(leaderAddress)); - - if (r->state == RAFT_LEADER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); - } else if (r->state == RAFT_FOLLOWER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); - } - - for (int j = 0; j < NODE_COUNT; j++) { - if (strcmp(leaderAddress, leaderStates[j].address) == 0) { - leaderStates[j].leaderCount++; - } - } - } -} - - -void raftTransferCb(struct raft_transfer *req) { - SRaftServer *pRaftServer = req->data; - raft_free(req); - - //printf("raftTransferCb: \n"); - updateLeaderStates(pRaftServer); - //printLeaderCount(); - - int myLeaderCount; - for (int i = 0; i < NODE_COUNT; ++i) { - if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { - myLeaderCount = leaderStates[i].leaderCount; - } - } - - //printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); - if (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { - struct raft *r; - for (int j = 0; j < pRaftServer->instanceCount; ++j) { - if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { - r = &pRaftServer->instance[j].raft; - break; - } - } - - struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); - transfer->data = pRaftServer; - - uint64_t destRaftId; - int minIndex = -1; - int minLeaderCount = myLeaderCount; - for (int j = 0; j < NODE_COUNT; ++j) { - if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) { - continue; - } - - if (leaderStates[j].leaderCount <= minLeaderCount) { - minLeaderCount = leaderStates[j].leaderCount; - minIndex = j; - } - } - - - char myHost[48]; - uint16_t myPort; - uint16_t myVid; - decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); - - - //printf("raftTransferCb transfer leader: vid[%u] choose: index:%d, leaderStates[%d].address:%s, leaderStates[%d].leaderCount:%d \n", minIndex, minIndex, leaderStates[minIndex].address, minIndex, leaderStates[minIndex].leaderCount); - - char *destAddress = leaderStates[minIndex].address; - - char tokens[MAX_PEERS][MAX_TOKEN_LEN]; - splitString(destAddress, ":", tokens, 2); - char *destHost = tokens[0]; - uint16_t destPort = atoi(tokens[1]); - destRaftId = encodeRaftId(destHost, destPort, myVid); - - printf("\nraftTransferCb transfer leader: vgroupId:%u from:%s:%u --> to:%s:%u ", myVid, myHost, myPort, destHost, destPort); - fflush(stdout); - - raft_transfer(r, transfer, destRaftId, raftTransferCb); - } - -} - - -void parseAddr(const char *addr, char *host, int len, uint32_t *port) { - char* tmp = (char*)malloc(strlen(addr) + 1); - strcpy(tmp, addr); - - char* context; - char* separator = ":"; - char* token = strtok_r(tmp, separator, &context); - if (token) { - snprintf(host, len, "%s", token); - } - - token = strtok_r(NULL, separator, &context); - if (token) { - sscanf(token, "%u", port); - } - - free(tmp); -} - -// only parse 3 tokens -int parseCommand3(const char* str, char* token1, char* token2, char* token3, int len) -{ - char* tmp = (char*)malloc(strlen(str) + 1); - strcpy(tmp, str); - - char* context; - char* separator = " "; - int n = 0; - - char* token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token1, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token2, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token3, token, len); - n++; - } - -ret: - return n; - free(tmp); -} - -// only parse 4 tokens -int parseCommand4(const char* str, char* token1, char* token2, char* token3, char *token4, int len) -{ - char* tmp = (char*)malloc(strlen(str) + 1); - strcpy(tmp, str); - - char* context; - char* separator = " "; - int n = 0; - - char* token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token1, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token2, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token3, token, len); - n++; - } - - token = strtok_r(NULL, separator, &context); - if (!token) { - goto ret; - } - if (strcmp(token, "") != 0) { - strncpy(token4, token, len); - n++; - } - -ret: - return n; - free(tmp); -} - -void *startServerFunc(void *param) { - SRaftServer *pServer = (SRaftServer*)param; - int32_t r = raftServerStart(pServer); - assert(r == 0); - - return NULL; -} - -// Console --------------------------------- -const char* state2String(unsigned short state) { - if (state == RAFT_UNAVAILABLE) { - return "RAFT_UNAVAILABLE"; - - } else if (state == RAFT_FOLLOWER) { - return "RAFT_FOLLOWER"; - - } else if (state == RAFT_CANDIDATE) { - return "RAFT_CANDIDATE"; - - } else if (state == RAFT_LEADER) { - return "RAFT_LEADER"; - - } - return "UNKNOWN_RAFT_STATE"; -} - - -void printRaftState2(struct raft *r) { - char leaderAddress[128]; - memset(leaderAddress, 0, sizeof(leaderAddress)); - - if (r->state == RAFT_LEADER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->address); - } else if (r->state == RAFT_FOLLOWER) { - snprintf(leaderAddress, sizeof(leaderAddress), "%s", r->follower_state.current_leader.address); - } - - for (int i = 0; i < r->configuration.n; ++i) { - char tmpAddress[128]; - snprintf(tmpAddress, sizeof(tmpAddress), "%s", r->configuration.servers[i].address); - - uint64_t raftId = r->configuration.servers[i].id; - char host[128]; - uint16_t port; - uint16_t vid; - decodeRaftId(raftId, host, 128, &port, &vid); - - char buf[512]; - memset(buf, 0, sizeof(buf)); - if (strcmp(tmpAddress, leaderAddress) == 0) { - snprintf(buf, sizeof(buf), "<%s:%u-%u-LEADER>\t", host, port, vid); - } else { - snprintf(buf, sizeof(buf), "<%s:%u-%u-FOLLOWER>\t", host, port, vid); - } - printf("%s", buf); - } - printf("\n"); -} - -void printRaftConfiguration(struct raft_configuration *c) { - printf("configuration: \n"); - for (int i = 0; i < c->n; ++i) { - printf("%llu -- %d -- %s\n", c->servers[i].id, c->servers[i].role, c->servers[i].address); - } -} - -void printRaftState(struct raft *r) { - printf("----Raft State: -----------\n"); - printf("mem_addr: %p \n", r); - printf("my_id: %llu \n", r->id); - printf("address: %s \n", r->address); - printf("current_term: %llu \n", r->current_term); - printf("voted_for: %llu \n", r->voted_for); - printf("role: %s \n", state2String(r->state)); - printf("commit_index: %llu \n", r->commit_index); - printf("last_applied: %llu \n", r->last_applied); - printf("last_stored: %llu \n", r->last_stored); - - printf("configuration_index: %llu \n", r->configuration_index); - printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index); - printRaftConfiguration(&r->configuration); - - printf("----------------------------\n"); -} - -void putValueCb(struct raft_apply *req, int status, void *result) { - raft_free(req); - struct raft *r = req->data; - if (status != 0) { - printf("putValueCb: %s \n", raft_errmsg(r)); - } else { - printf("putValueCb: %s \n", "ok"); - } -} - -void putValue(struct raft *r, const char *value) { - struct raft_buffer buf; - - buf.len = TOKEN_LEN;; - buf.base = raft_malloc(buf.len); - snprintf(buf.base, buf.len, "%s", value); - - struct raft_apply *req = raft_malloc(sizeof(struct raft_apply)); - req->data = r; - int ret = raft_apply(r, req, &buf, 1, putValueCb); - if (ret == 0) { - printf("put %s \n", (char*)buf.base); - } else { - printf("put error: %s \n", raft_errmsg(r)); - } -} - -void getValue(const char *key) { - char *ptr = getKV(key); - if (ptr) { - printf("get value: [%s] \n", ptr); - } else { - printf("value not found for key: [%s] \n", key); - } -} - -void console(SRaftServer *pRaftServer) { - while (1) { - char cmd_buf[COMMAND_LEN]; - memset(cmd_buf, 0, sizeof(cmd_buf)); - printf("(console)> "); - char *ret = fgets(cmd_buf, COMMAND_LEN, stdin); - if (!ret) { - exit(-1); - } - - int pos = strlen(cmd_buf); - if(cmd_buf[pos - 1] == '\n') { - cmd_buf[pos - 1] = '\0'; - } - - if (strncmp(cmd_buf, "", COMMAND_LEN) == 0) { - continue; - } - - char cmd[TOKEN_LEN]; - memset(cmd, 0, sizeof(cmd)); - - char param1[TOKEN_LEN]; - memset(param1, 0, sizeof(param1)); - - char param2[TOKEN_LEN]; - memset(param2, 0, sizeof(param2)); - - char param3[TOKEN_LEN]; - memset(param2, 0, sizeof(param2)); - - parseCommand4(cmd_buf, cmd, param1, param2, param3, TOKEN_LEN); - if (strcmp(cmd, "addnode") == 0) { - printf("not support \n"); - - /* - char host[HOST_LEN]; - uint32_t port; - parseAddr(param1, host, HOST_LEN, &port); - uint64_t rid = raftId(host, port); - - struct raft_change *req = raft_malloc(sizeof(*req)); - int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL); - if (r != 0) { - printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft)); - } - printf("add node: %lu %s \n", rid, param1); - - struct raft_change *req2 = raft_malloc(sizeof(*req2)); - r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL); - if (r != 0) { - printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft)); - } - */ - - } else if (strcmp(cmd, "dropnode") == 0) { - printf("not support \n"); - - } else if (strcmp(cmd, "quit") == 0 || strcmp(cmd, "exit") == 0) { - exit(0); - - } else if (strcmp(cmd, "rebalance") == 0 && strcmp(param1, "leader") == 0) { - - /* - updateLeaderStates(pRaftServer); - - int myLeaderCount; - for (int i = 0; i < NODE_COUNT; ++i) { - if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { - myLeaderCount = leaderStates[i].leaderCount; - } - } - - while (myLeaderCount > pRaftServer->instanceCount / NODE_COUNT) { - printf("myLeaderCount:%d waterLevel:%d \n", myLeaderCount, pRaftServer->instanceCount / NODE_COUNT); - - struct raft *r; - for (int j = 0; j < pRaftServer->instanceCount; ++j) { - if (pRaftServer->instance[j].raft.state == RAFT_LEADER) { - r = &pRaftServer->instance[j].raft; - } - } - - struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); - transfer->data = pRaftServer; - - uint64_t destRaftId; - int minIndex = -1; - int minLeaderCount = myLeaderCount; - for (int j = 0; j < NODE_COUNT; ++j) { - if (strcmp(leaderStates[j].address, pRaftServer->address) == 0) continue; - - printf("-----leaderStates[%d].leaderCount:%d \n", j, leaderStates[j].leaderCount); - if (leaderStates[j].leaderCount <= minLeaderCount) { - minIndex = j; - printf("++++ assign minIndex : %d \n", minIndex); - } - } - - printf("minIndex:%d minLeaderCount:%d \n", minIndex, minLeaderCount); - - char myHost[48]; - uint16_t myPort; - uint16_t myVid; - decodeRaftId(r->id, myHost, sizeof(myHost), &myPort, &myVid); - - char *destAddress = leaderStates[minIndex].address; - - char tokens[MAX_PEERS][MAX_TOKEN_LEN]; - splitString(destAddress, ":", tokens, 2); - char *destHost = tokens[0]; - uint16_t destPort = atoi(tokens[1]); - destRaftId = encodeRaftId(destHost, destPort, myVid); - - printf("destHost:%s destPort:%u myVid:%u", destHost, destPort, myVid); - raft_transfer(r, transfer, destRaftId, raftTransferCb); - sleep(1); - - for (int i = 0; i < NODE_COUNT; ++i) { - if (strcmp(pRaftServer->address, leaderStates[i].address) == 0) { - myLeaderCount = leaderStates[i].leaderCount; - } - } - } - */ - - - int leaderCount = 0; - - struct raft *firstR; - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - struct raft *r = &pRaftServer->instance[i].raft; - if (r->state == RAFT_LEADER) { - leaderCount++; - firstR = r; - } - } - - if (leaderCount > pRaftServer->instanceCount / NODE_COUNT) { - struct raft_transfer *transfer = raft_malloc(sizeof(*transfer)); - transfer->data = pRaftServer; - raft_transfer(firstR, transfer, 0, raftTransferCb); - } - - - } else if (strcmp(cmd, "put") == 0) { - char buf[256]; - uint16_t vid; - sscanf(param1, "%hu", &vid); - snprintf(buf, sizeof(buf), "%s--%s", param2, param3); - putValue(&pRaftServer->instance[vid].raft, buf); - - } else if (strcmp(cmd, "get") == 0) { - getValue(param1); - - } else if (strcmp(cmd, "transfer") == 0) { - uint16_t vid; - sscanf(param1, "%hu", &vid); - - struct raft_transfer transfer; - raft_transfer(&pRaftServer->instance[vid].raft, &transfer, 0, NULL); - - - } else if (strcmp(cmd, "state") == 0) { - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - printf("instance %d: ", i); - printRaftState(&pRaftServer->instance[i].raft); - } - - } else if (strcmp(cmd, "leader") == 0 && strcmp(param1, "state") == 0) { - updateLeaderStates(pRaftServer); - printf("\n--------------------------------------------\n"); - printLeaderCount(); - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - printRaftState2(&pRaftServer->instance[i].raft); - } - printf("--------------------------------------------\n"); - - } else if (strcmp(cmd, "snapshot") == 0) { - printf("not support \n"); - - } else if (strcmp(cmd, "help") == 0) { - printf("addnode \"127.0.0.1:8888\" \n"); - printf("dropnode \"127.0.0.1:8888\" \n"); - printf("put key value \n"); - printf("get key \n"); - printf("state \n"); - - } else { - printf("unknown command: [%s], type \"help\" to see help \n", cmd); - } - - //printf("cmd_buf: [%s] \n", cmd_buf); - } -} - -void *startConsoleFunc(void *param) { - SRaftServer *pServer = (SRaftServer*)param; - console(pServer); - return NULL; -} - -// Config --------------------------------- -void usage() { - printf("\nusage: \n"); - printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name); - printf("\n"); - printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name); - printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name); - printf("%s --me=127.0.0.1:10002 --peers=127.0.0.1:10000,127.0.0.1:10001 --dir=./data \n", exe_name); - printf("\n"); -} - -void parseConf(int argc, char **argv, SRaftServerConfig *pConf) { - memset(pConf, 0, sizeof(*pConf)); - - int option_index, option_value; - option_index = 0; - static struct option long_options[] = { - {"help", no_argument, NULL, 'h'}, - {"peers", required_argument, NULL, 'p'}, - {"me", required_argument, NULL, 'm'}, - {"dir", required_argument, NULL, 'd'}, - {NULL, 0, NULL, 0} - }; - - while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) { - switch (option_value) { - case 'm': { - parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port); - break; - } - - case 'p': { - char tokens[MAX_PEERS][MAX_TOKEN_LEN]; - int peerCount = splitString(optarg, ",", tokens, MAX_PEERS); - pConf->peersCount = peerCount; - for (int i = 0; i < peerCount; ++i) { - Addr *pAddr = &pConf->peers[i]; - parseAddr(tokens[i], pAddr->host, sizeof(pAddr->host), &pAddr->port); - } - break; - } - - - case 'd': { - snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg); - break; - } - - case 'h': { - usage(); - exit(-1); - } - - default: { - usage(); - exit(-1); - } - } - } - snprintf(pConf->dataDir, sizeof(pConf->dataDir), "%s/%s_%u", pConf->dir, pConf->me.host, pConf->me.port); -} - -void printConf(SRaftServerConfig *pConf) { - printf("\nconf: \n"); - printf("me: %s:%u \n", pConf->me.host, pConf->me.port); - printf("peersCount: %d \n", pConf->peersCount); - for (int i = 0; i < pConf->peersCount; ++i) { - Addr *pAddr = &pConf->peers[i]; - printf("peer%d: %s:%u \n", i, pAddr->host, pAddr->port); - } - printf("dataDir: %s \n\n", pConf->dataDir); - -} - - -int main(int argc, char **argv) { - srand(time(NULL)); - int32_t ret; - - exe_name = argv[0]; - if (argc < 3) { - usage(); - exit(-1); - } - - SRaftServerConfig conf; - parseConf(argc, argv, &conf); - printConf(&conf); - - signal(SIGPIPE, SIG_IGN); - - /* - char cmd_buf[COMMAND_LEN]; - snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", conf.dataDir); - system(cmd_buf); - */ - - - struct raft_fsm fsm; - initFsm(&fsm); - - SRaftServer raftServer; - ret = raftServerInit(&raftServer, &conf, &fsm); - assert(ret == 0); - - pthread_t tidRaftServer; - pthread_create(&tidRaftServer, NULL, startServerFunc, &raftServer); - - pthread_t tidConsole; - pthread_create(&tidConsole, NULL, startConsoleFunc, &raftServer); - - while (1) { - sleep(10); - } - - return 0; -} diff --git a/contrib/test/traft/rebalance_leader/raftServer.c b/contrib/test/traft/rebalance_leader/raftServer.c deleted file mode 100644 index 165d3c9023..0000000000 --- a/contrib/test/traft/rebalance_leader/raftServer.c +++ /dev/null @@ -1,224 +0,0 @@ -#include -#include -#include "common.h" -#include "raftServer.h" - -//char *keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);; -//char *values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN); - - -char keys[MAX_KV_LEN][MAX_RECORD_COUNT]; -char values[MAX_KV_LEN][MAX_RECORD_COUNT]; -int writeIndex = 0; - -void initStore() { -} - -void destroyStore() { - //free(keys); - //free(values); -} - -void putKV(const char *key, const char *value) { - if (writeIndex < MAX_RECORD_COUNT) { - strncpy(keys[writeIndex], key, MAX_KV_LEN); - strncpy(values[writeIndex], value, MAX_KV_LEN); - writeIndex++; - } -} - -char *getKV(const char *key) { - for (int i = 0; i < MAX_RECORD_COUNT; ++i) { - if (strcmp(keys[i], key) == 0) { - return values[i]; - } - } - return NULL; -} - - -int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr) -{ - if (n_arr <= 0) { - return -1; - } - - char* tmp = (char*)malloc(strlen(str) + 1); - strcpy(tmp, str); - char* context; - int n = 0; - - char* token = strtok_r(tmp, separator, &context); - if (!token) { - goto ret; - } - strncpy(arr[n], token, MAX_TOKEN_LEN); - n++; - - while (1) { - token = strtok_r(NULL, separator, &context); - if (!token || n >= n_arr) { - goto ret; - } - strncpy(arr[n], token, MAX_TOKEN_LEN); - n++; - } - -ret: - free(tmp); - return n; -} - -/* -uint64_t raftId(const char *host, uint32_t port) { - uint32_t host_uint32 = (uint32_t)inet_addr(host); - assert(host_uint32 != (uint32_t)-1); - uint64_t code = ((uint64_t)host_uint32) << 32 | port; - return code; -} -*/ - - -/* -uint64_t encodeRaftId(const char *host, uint16_t port, uint16_t vid) { - uint64_t raftId; - uint32_t host_uint32 = (uint32_t)inet_addr(host); - assert(host_uint32 != (uint32_t)-1); - - raftId = (((uint64_t)host_uint32) << 32) | (((uint32_t)port) << 16) | vid; - return raftId; -} - -void decodeRaftId(uint64_t raftId, char *host, int32_t len, uint16_t *port, uint16_t *vid) { - uint32_t host32 = (uint32_t)((raftId >> 32) & 0x00000000FFFFFFFF); - - struct in_addr addr; - addr.s_addr = host32; - snprintf(host, len, "%s", inet_ntoa(addr)); - - *port = (uint16_t)((raftId & 0x00000000FFFF0000) >> 16); - *vid = (uint16_t)(raftId & 0x000000000000FFFF); -} -*/ - - - - -int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm) { - int ret; - - snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host); - pRaftServer->port = pConf->me.port; - snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port); - //strncpy(pRaftServer->dir, pConf->dataDir, sizeof(pRaftServer->dir)); - - ret = uv_loop_init(&pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "uv_loop_init error: %s \n", uv_strerror(ret)); - assert(0); - } - - ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop); - if (ret != 0) { - fprintf(stderr, "raft_uv_tcp_init: error %d \n", ret); - assert(0); - } - - - uint16_t vid; - pRaftServer->instanceCount = 20; - - - for (int i = 0; i < pRaftServer->instanceCount; ++i) - { - //vid = 0; - vid = i; - - - pRaftServer->instance[vid].raftId = encodeRaftId(pRaftServer->host, pRaftServer->port, vid); - snprintf(pRaftServer->instance[vid].dir, sizeof(pRaftServer->instance[vid].dir), "%s_%llu", pConf->dataDir, pRaftServer->instance[vid].raftId); - - char cmd_buf[COMMAND_LEN]; - snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", pRaftServer->instance[vid].dir); - system(cmd_buf); - sleep(1); - - pRaftServer->instance[vid].fsm = pFsm; - - ret = raft_uv_init(&pRaftServer->instance[vid].io, &pRaftServer->loop, pRaftServer->instance[vid].dir, &pRaftServer->transport); - if (ret != 0) { - fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); - assert(0); - } - - ret = raft_init(&pRaftServer->instance[vid].raft, &pRaftServer->instance[vid].io, pRaftServer->instance[vid].fsm, pRaftServer->instance[vid].raftId, pRaftServer->address); - if (ret != 0) { - fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[vid].raft)); - assert(0); - } - - struct raft_configuration conf; - raft_configuration_init(&conf); - raft_configuration_add(&conf, pRaftServer->instance[vid].raftId, pRaftServer->address, RAFT_VOTER); - printf("add myself: %llu - %s \n", pRaftServer->instance[vid].raftId, pRaftServer->address); - for (int i = 0; i < pConf->peersCount; ++i) { - const Addr *pAddr = &pConf->peers[i]; - - raft_id rid = encodeRaftId(pAddr->host, pAddr->port, vid); - - char addrBuf[ADDRESS_LEN]; - snprintf(addrBuf, sizeof(addrBuf), "%s:%u", pAddr->host, pAddr->port); - raft_configuration_add(&conf, rid, addrBuf, RAFT_VOTER); - printf("add peers: %llu - %s \n", rid, addrBuf); - } - - raft_bootstrap(&pRaftServer->instance[vid].raft, &conf); - - } - - - - - - - - return 0; -} - -int32_t raftServerStart(SRaftServer *pRaftServer) { - int ret; - - for (int i = 0; i < pRaftServer->instanceCount; ++i) { - ret = raft_start(&pRaftServer->instance[i].raft); - if (ret != 0) { - fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->instance[i].raft)); - } - - } - - - uv_run(&pRaftServer->loop, UV_RUN_DEFAULT); -} - - -void raftServerClose(SRaftServer *pRaftServer) { - -} - - -int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) { - char *msg = (char*)buf->base; - printf("fsm apply: %s \n", msg); - - char arr[2][MAX_TOKEN_LEN]; - splitString(msg, "--", arr, 2); - putKV(arr[0], arr[1]); - - return 0; -} - -int32_t initFsm(struct raft_fsm *fsm) { - initStore(); - fsm->apply = fsmApplyCb; - return 0; -} diff --git a/contrib/test/traft/rebalance_leader/raftServer.h b/contrib/test/traft/rebalance_leader/raftServer.h deleted file mode 100644 index 5ea43985c9..0000000000 --- a/contrib/test/traft/rebalance_leader/raftServer.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef TDENGINE_RAFT_SERVER_H -#define TDENGINE_RAFT_SERVER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include "raft.h" -#include "raft/uv.h" -#include "common.h" - - -// simulate a db store, just for test -#define MAX_KV_LEN 20 -#define MAX_RECORD_COUNT 16 - - -//char *keys; -//char *values; -//int writeIndex; - -void initStore(); -void destroyStore(); -void putKV(const char *key, const char *value); -char *getKV(const char *key); - -typedef struct { - char dir[DIR_LEN + HOST_LEN * 4]; /* Data dir of UV I/O backend */ - raft_id raftId; /* For vote */ - struct raft_fsm *fsm; /* Sample application FSM */ - struct raft raft; /* Raft instance */ - struct raft_io io; /* UV I/O backend */ - -} SInstance; - -typedef struct { - char host[HOST_LEN]; - uint32_t port; - char address[ADDRESS_LEN]; /* Raft instance address */ - - struct uv_loop_s loop; /* UV loop */ - struct raft_uv_transport transport; /* UV I/O backend transport */ - - SInstance instance[MAX_INSTANCE_NUM]; - int32_t instanceCount; - -} SRaftServer; - -#define MAX_TOKEN_LEN 32 -int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr); - -int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm); -int32_t raftServerStart(SRaftServer *pRaftServer); -void raftServerClose(SRaftServer *pRaftServer); - - -int initFsm(struct raft_fsm *fsm); - - - - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RAFT_SERVER_H diff --git a/contrib/test/traft/single_node/CMakeLists.txt b/contrib/test/traft/single_node/CMakeLists.txt new file mode 100644 index 0000000000..666ce271b8 --- /dev/null +++ b/contrib/test/traft/single_node/CMakeLists.txt @@ -0,0 +1,6 @@ +add_executable(singleNode "") +target_sources(singleNode + PRIVATE + "singleNode.c" +) +target_link_libraries(singleNode PUBLIC traft lz4 uv_a) diff --git a/contrib/test/traft/single_node/clear.sh b/contrib/test/traft/single_node/clear.sh new file mode 100644 index 0000000000..398b3088f2 --- /dev/null +++ b/contrib/test/traft/single_node/clear.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +rm -rf 127.0.0.1* +rm -rf ./data diff --git a/contrib/test/traft/single_node/cmd b/contrib/test/traft/single_node/cmd new file mode 100644 index 0000000000..ff8a156f64 --- /dev/null +++ b/contrib/test/traft/single_node/cmd @@ -0,0 +1,6 @@ +all: + gcc singleNode.c -I ../../include/ ../../.libs/libraft.a -o singleNode -luv -llz4 -lpthread -g +clean: + rm -f singleNode + sh clear.sh + diff --git a/contrib/test/traft/single_node/singleNode.c b/contrib/test/traft/single_node/singleNode.c new file mode 100644 index 0000000000..d22af30e45 --- /dev/null +++ b/contrib/test/traft/single_node/singleNode.c @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +//#include + +#include "raft.h" + +SRaftEnv raftEnv; + +typedef struct Tsdb { + uint64_t lastApplyIndex; + void *mem; + void *imm; + void *store; +} Tsdb; + +void tsdbWrite(Tsdb *t, char *msg) {} + +void *startFunc(void *param) { + SRaftEnv *pSRaftEnv = (SRaftEnv *)param; + int32_t r = raftEnvStart(pSRaftEnv); + assert(r == 0); + return NULL; +} + +int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result, raft_index index) { + // get commit value + char *msg = (char *)buf->base; + printf("fsm apply: index:%llu value:%s \n", index, msg); + + Tsdb *t = pFsm->data; + if (index > t->lastApplyIndex) { + // apply value into tsdb + tsdbWrite(t, msg); + + // update lastApplyIndex + t->lastApplyIndex = index; + } + + return 0; +} + +void putValueCb(struct raft_apply *req, int status, void *result) { + void *ptr = req->data; + if (status != 0) { + printf("putValueCb error \n"); + } else { + printf("putValueCb ok \n"); + } + free(ptr); + free(req); +} + +void submitValue() { + // prepare value + struct raft_buffer buf; + buf.len = 32; + void *ptr = malloc(buf.len); + buf.base = ptr; + snprintf(buf.base, buf.len, "%ld", time(NULL)); + + // get raft + struct raft *r = getRaft(&raftEnv, 100); + assert(r != NULL); + // printRaftState(r); + + // submit value + struct raft_apply *req = malloc(sizeof(*req)); + req->data = ptr; + int ret = raft_apply(r, req, &buf, 1, putValueCb); + if (ret == 0) { + printf("put %s \n", (char *)buf.base); + } else { + printf("put error: %s \n", raft_errmsg(r)); + } +} + +int main(int argc, char **argv) { + // init raft env + int r = raftEnvInit(&raftEnv, "127.0.0.1", 38000, "./data"); + assert(r == 0); + + // start raft env + pthread_t tid; + pthread_create(&tid, NULL, startFunc, &raftEnv); + + // wait for start, just for simple + while (raftEnv.isStart != 1) { + sleep(1); + } + + // init fsm + struct raft_fsm *pFsm = malloc(sizeof(*pFsm)); + pFsm->apply = fsmApplyCb; + Tsdb *tsdb = malloc(sizeof(*tsdb)); + pFsm->data = tsdb; + + // add vgroup, id = 100, only has one replica + r = addRaftVoter(&raftEnv, NULL, 0, 100, pFsm); + assert(r == 0); + + // for test: submit a value every second + while (1) { + sleep(1); + submitValue(); + } + + return 0; +} diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b468456cb7..857de3671f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -696,13 +696,13 @@ typedef struct SVgroupInfo { uint32_t hashEnd; int8_t inUse; int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; typedef struct { int32_t vgId; int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SVgroupMsg; typedef struct { @@ -1529,17 +1529,22 @@ typedef struct SMqSetCVgReq { } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { - int32_t tlen = sizeof(SSubQueryMsg) + pMsg->contentLen; - if (buf == NULL) return tlen; - memcpy(*buf, pMsg, tlen); - *buf = POINTER_SHIFT(*buf, tlen); + int32_t tlen = 0; + tlen += taosEncodeFixedU64(buf, pMsg->sId); + tlen += taosEncodeFixedU64(buf, pMsg->queryId); + tlen += taosEncodeFixedU64(buf, pMsg->taskId); + tlen += taosEncodeFixedU32(buf, pMsg->contentLen); + tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { - int32_t tlen = sizeof(SSubQueryMsg) + ((SSubQueryMsg*)buf)->contentLen; - memcpy(pMsg, buf, tlen); - return POINTER_SHIFT(buf, tlen); + buf = taosDecodeFixedU64(buf, &pMsg->sId); + buf = taosDecodeFixedU64(buf, &pMsg->queryId); + buf = taosDecodeFixedU64(buf, &pMsg->taskId); + buf = taosDecodeFixedU32(buf, &pMsg->contentLen); + buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); + return buf; } static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { @@ -1574,32 +1579,47 @@ typedef struct SMqSetCVgRsp { char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; -typedef struct SMqCVConsumeReq { +typedef struct SMqConsumeReq { int64_t reqId; int64_t offset; int64_t consumerId; int64_t blockingTime; char topicName[TSDB_TOPIC_FNAME_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN]; -} SMqCVConsumeReq; +} SMqConsumeReq; -typedef struct SMqConsumeRspBlock { - int32_t bodyLen; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char body[]; -} SMqConsumeRspBlock; +typedef struct SMqColData { + int16_t colId; + int16_t type; + int16_t bytes; + char data[]; +} SMqColData; -typedef struct SMqCVConsumeRsp { - int64_t reqId; - int64_t clientId; - int64_t committedOffset; - int64_t receiveOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t bodyLen; - char topicName[TSDB_TOPIC_FNAME_LEN]; - SMqConsumeRspBlock blocks[]; -} SMqCvConsumeRsp; +typedef struct SMqTbData { + int64_t uid; + int32_t numOfCols; + int32_t numOfRows; + SMqColData colData[]; +} SMqTbData; + +typedef struct SMqTopicBlk { + char topicName[TSDB_TOPIC_FNAME_LEN]; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t bodyLen; + int32_t numOfTb; + SMqTbData tbData[]; +} SMqTopicData; + +typedef struct SMqConsumeRsp { + int64_t reqId; + int64_t clientId; + int32_t bodyLen; + int32_t numOfTopics; + SMqTopicData data[]; +} SMqConsumeRsp; #ifdef __cplusplus } diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 61970ff440..457245e9a3 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -26,17 +26,26 @@ typedef void* qTaskInfo_t; typedef void* DataSinkHandle; struct SSubplan; + /** + * Create the exec task for streaming mode + * @param pMsg + * @param streamReadHandle + * @return + */ +qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* streamReadHandle); + +int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); /** * Create the exec task object according to task json - * @param tsdb + * @param readHandle * @param vgId * @param pTaskInfoMsg * @param pTaskInfo * @param qId * @return */ -int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +int32_t qCreateExecTask(void* readHandle, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, @@ -53,63 +62,63 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds); * this function will be blocked to wait for the query execution completed or paused, * in which case enough results have been produced already. * - * @param qinfo + * @param tinfo * @return */ -int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext); +int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext); /** * * Retrieve the actual results to fill the response message payload. * Note that this function must be executed after qRetrieveQueryResultInfo is invoked. * - * @param qinfo qinfo object + * @param tinfo tinfo object * @param pRsp response message * @param contLen payload length * @return */ -//int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +//int32_t qDumpRetrieveResult(qTaskInfo_t tinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) - * @param qinfo + * @param tinfo * @return */ -void* qGetResultRetrieveMsg(qTaskInfo_t qinfo); +void* qGetResultRetrieveMsg(qTaskInfo_t tinfo); /** * kill the ongoing query and free the query handle and corresponding resources automatically - * @param qinfo qhandle + * @param tinfo qhandle * @return */ -int32_t qKillTask(qTaskInfo_t qinfo); +int32_t qKillTask(qTaskInfo_t tinfo); /** * kill the ongoing query asynchronously - * @param qinfo qhandle + * @param tinfo qhandle * @return */ -int32_t qAsyncKillTask(qTaskInfo_t qinfo); +int32_t qAsyncKillTask(qTaskInfo_t tinfo); /** * return whether query is completed or not - * @param qinfo + * @param tinfo * @return */ -int32_t qIsTaskCompleted(qTaskInfo_t qinfo); +int32_t qIsTaskCompleted(qTaskInfo_t tinfo); /** * destroy query info structure * @param qHandle */ -void qDestroyTask(qTaskInfo_t qHandle); +void qDestroyTask(qTaskInfo_t tinfo); /** * Get the queried table uid * @param qHandle * @return */ -int64_t qGetQueriedTableUid(qTaskInfo_t qHandle); +int64_t qGetQueriedTableUid(qTaskInfo_t tinfo); /** * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. @@ -136,7 +145,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param type operation type: ADD|DROP * @return */ -int32_t qUpdateQueriedTableIdList(qTaskInfo_t qinfo, int64_t uid, int32_t type); +int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type); //================================================================================================ // query handle management @@ -203,4 +212,4 @@ void** qDeregisterQInfo(void* pMgmt, void* pQInfo); } #endif -#endif /*_TD_EXECUTOR_H_*/ \ No newline at end of file +#endif /*_TD_EXECUTOR_H_*/ diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index ba3539e64e..126cee390c 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -178,7 +178,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); + int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/include/util/compare.h b/include/util/compare.h index 461552ca7b..70a8134b35 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_COMPARE_H -#define _TD_UTIL_COMPARE_H +#ifndef _TD_UTIL_COMPARE_H_ +#define _TD_UTIL_COMPARE_H_ #ifdef __cplusplus extern "C" { @@ -35,67 +35,65 @@ extern "C" { #define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) #define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) -#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' } +#define PATTERN_COMPARE_INFO_INITIALIZER \ + { '%', '_' } typedef struct SPatternCompareInfo { char matchAll; // symbol for match all wildcard, default: '%' char matchOne; // symbol for match one wildcard, default: '_' } SPatternCompareInfo; -int patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo); -int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); +int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo); - -int32_t taosArrayCompareString(const void* a, const void* b); +int32_t taosArrayCompareString(const void *a, const void *b); int32_t setCompareBytes1(const void *pLeft, const void *pRight); - int32_t setCompareBytes2(const void *pLeft, const void *pRight); - int32_t setCompareBytes4(const void *pLeft, const void *pRight); int32_t setCompareBytes8(const void *pLeft, const void *pRight); +int32_t compareInt8Val(const void *pLeft, const void *pRight); +int32_t compareInt16Val(const void *pLeft, const void *pRight); int32_t compareInt32Val(const void *pLeft, const void *pRight); int32_t compareInt64Val(const void *pLeft, const void *pRight); -int32_t compareInt16Val(const void *pLeft, const void *pRight); - -int32_t compareInt8Val(const void *pLeft, const void *pRight); - +int32_t compareUint8Val(const void *pLeft, const void *pRight); +int32_t compareUint16Val(const void *pLeft, const void *pRight); int32_t compareUint32Val(const void *pLeft, const void *pRight); int32_t compareUint64Val(const void *pLeft, const void *pRight); -int32_t compareUint16Val(const void *pLeft, const void *pRight); - -int32_t compareUint8Val(const void* pLeft, const void* pRight); - int32_t compareFloatVal(const void *pLeft, const void *pRight); - int32_t compareDoubleVal(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight); - int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight); -int32_t compareStrRegexComp(const void* pLeft, const void* pRight); -int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight); -int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight); -int32_t compareFindItemInSet(const void *pLeft, const void* pRight); + +int32_t compareStrRegexComp(const void *pLeft, const void *pRight); +int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); +int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); + +int32_t compareFindItemInSet(const void *pLeft, const void *pRight); + int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); -int32_t compareInt16ValDesc(const void* pLeft, const void* pRight); -int32_t compareInt32ValDesc(const void* pLeft, const void* pRight); -int32_t compareInt64ValDesc(const void* pLeft, const void* pRight); -int32_t compareFloatValDesc(const void* pLeft, const void* pRight); -int32_t compareDoubleValDesc(const void* pLeft, const void* pRight); -int32_t compareUint8ValDesc(const void* pLeft, const void* pRight); -int32_t compareUint16ValDesc(const void* pLeft, const void* pRight); -int32_t compareUint32ValDesc(const void* pLeft, const void* pRight); -int32_t compareUint64ValDesc(const void* pLeft, const void* pRight); -int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight); -int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight); +int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); +int32_t compareInt32ValDesc(const void *pLeft, const void *pRight); +int32_t compareInt64ValDesc(const void *pLeft, const void *pRight); + +int32_t compareFloatValDesc(const void *pLeft, const void *pRight); +int32_t compareDoubleValDesc(const void *pLeft, const void *pRight); + +int32_t compareUint8ValDesc(const void *pLeft, const void *pRight); +int32_t compareUint16ValDesc(const void *pLeft, const void *pRight); +int32_t compareUint32ValDesc(const void *pLeft, const void *pRight); +int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); + +int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); +int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_COMPARE_H*/ +#endif /*_TD_UTIL_COMPARE_H_*/ diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 8198787048..c105ce1ab9 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -372,7 +372,7 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { } // ---- binary -static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valueLen) { +static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { int tlen = 0; if (buf != NULL) { @@ -384,14 +384,19 @@ static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int valu return tlen; } -static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int valueLen) { - uint64_t size = 0; +static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valueLen) { *value = malloc((size_t)valueLen); if (*value == NULL) return NULL; - memcpy(*value, buf, (size_t)size); + memcpy(*value, buf, (size_t)valueLen); - return POINTER_SHIFT(buf, size); + return POINTER_SHIFT(buf, valueLen); +} + +static FORCE_INLINE void *taosDecodeBinaryTo(void *buf, void *value, int32_t valueLen) { + + memcpy(value, buf, (size_t)valueLen); + return POINTER_SHIFT(buf, valueLen); } #endif diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 1d10869e30..296fbba634 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -136,6 +136,7 @@ typedef struct SReqResultInfo { TAOS_ROW row; char **pCol; uint32_t numOfRows; + uint64_t totalRows; uint32_t current; bool completed; } SReqResultInfo; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f747ccf3b6..0780593e7e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -110,7 +110,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.user = (char *)user; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; -// rpcInit.spi = 1; + rpcInit.spi = 1; rpcInit.secret = (char *)auth; void* pDnodeConn = rpcOpen(&rpcInit); @@ -196,6 +196,10 @@ static void doDestroyRequest(void* p) { doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryDag(pRequest->body.pDag); + if (pRequest->body.showInfo.pArray != NULL) { + taosArrayDestroy(pRequest->body.showInfo.pArray); + } + deregisterRequest(pRequest); tfree(pRequest); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 159a92b0ab..9a0a6a11e9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,9 +25,9 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); +static void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); - static bool stringLengthCheck(const char* str, size_t maxsize) { +static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { return false; } @@ -59,7 +59,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i } static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); -static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema); +static void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { if (taos_init() != TSDB_CODE_SUCCESS) { @@ -117,7 +117,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - p->pAppHbMgr = appHbMgrInit(p); + /*p->pAppHbMgr = appHbMgrInit(p);*/ taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -202,43 +202,38 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { +int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, SArray* pNodeList) { pRequest->type = pQueryNode->type; - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + SSchema* pSchema = NULL; + int32_t numOfCols = 0; + int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &numOfCols, pNodeList, pRequest->requestId); if (code != 0) { return code; } if (pQueryNode->type == TSDB_SQL_SELECT) { - SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0); - - SSubplan* pPlan = taosArrayGetP(pa, 0); - SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); - setResSchemaInfo(pResInfo, pDataBlockSchema); - + setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); pRequest->type = TDMT_VND_QUERY; } return code; } -void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) { - assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { + assert(pSchema != NULL && numOfCols > 0); - pResInfo->numOfCols = pDataBlockSchema->numOfCols; - pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0])); + pResInfo->numOfCols = numOfCols; + pResInfo->fields = calloc(numOfCols, sizeof(pSchema[0])); for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { - SSchema* pSchema = &pDataBlockSchema->pSchema[i]; - pResInfo->fields[i].bytes = pSchema->bytes; - pResInfo->fields[i].type = pSchema->type; - tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name)); + pResInfo->fields[i].bytes = pSchema[i].bytes; + pResInfo->fields[i].type = pSchema[i].type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); } } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { +int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; @@ -256,14 +251,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return pRequest->code; } - SArray *execNode = taosArrayInit(4, sizeof(SQueryNodeAddr)); - - SQueryNodeAddr addr = {.numOfEps = 1, .inUse = 0, .nodeId = 2}; - addr.epAddr[0].port = 7100; - strcpy(addr.epAddr[0].fqdn, "localhost"); - - taosArrayPush(execNode, &addr); - return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, execNode, pDag, &pRequest->body.pQueryJob); + return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); } typedef struct tmq_t tmq_t; @@ -399,7 +387,9 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, // todo check for invalid sql statement and return with error code - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return); + SSchema *schema = NULL; + int32_t numOfCols = 0; + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); pStr = qDagToString(pRequest->body.pDag); if(pStr == NULL) { @@ -492,6 +482,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQueryNode = NULL; + SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -500,8 +491,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return); + CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); pRequest->code = terrno; } @@ -571,7 +562,7 @@ STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uin tsem_wait(&pRequest->body.rspSem); if (pRequest->code != TSDB_CODE_SUCCESS) { - const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); printf("failed to connect to server, reason: %s\n\n", errorMsg); destroyRequest(pRequest); @@ -719,13 +710,17 @@ void* doFetchRow(SRequestObj* pRequest) { return NULL; } - int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData); if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; return NULL; } - setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData); + tscDebug("0x%"PRIx64 " fetch results, numOfRows:%d total Rows:%"PRId64", complete:%d, reqId:0x%"PRIx64, pRequest->self, pResInfo->numOfRows, + pResInfo->totalRows, pResInfo->completed, pRequest->requestId); + if (pResultInfo->numOfRows == 0) { return NULL; } @@ -855,7 +850,7 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } -void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { +void setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { assert(pResultInfo != NULL && pRsp != NULL); pResultInfo->pRspMsg = (const char*) pRsp; @@ -864,5 +859,6 @@ void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* p pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); + pResultInfo->totalRows += pResultInfo->numOfRows; setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 02e36043dc..81ea18fe08 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); + /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/ // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, @@ -145,19 +145,23 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { } pSchema = pMetaMsg->pSchema; - TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); - for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { - tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); - pFields[i].type = pSchema[i].type; - pFields[i].bytes = pSchema[i].bytes; - } + tfree(pRequest->body.resInfo.pRspMsg); pRequest->body.resInfo.pRspMsg = pMsg->pData; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - pResInfo->fields = pFields; - pResInfo->numOfCols = pMetaMsg->numOfColumns; + if (pResInfo->fields == NULL) { + TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); + pFields[i].type = pSchema[i].type; + pFields[i].bytes = pSchema[i].bytes; + } + pResInfo->fields = pFields; + } + + pResInfo->numOfCols = pMetaMsg->numOfColumns; pRequest->body.showInfo.execId = pShow->showId; // todo diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 736a47273f..ee23567705 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -17,8 +17,9 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -451,39 +452,39 @@ TEST(testCase, driverInit_Test) { // // taos_close(pConn); //} -// -//TEST(testCase, show_table_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "show tables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// } -// -// pRes = taos_query(pConn, "show abc1.tables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// int32_t count = 0; -// char str[512] = {0}; -// -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%d: %s\n", ++count, str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, show_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "show abc1.tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + int32_t count = 0; + char str[512] = {0}; + + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%d: %s\n", ++count, str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, drop_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -525,29 +526,29 @@ TEST(testCase, driverInit_Test) { // taosHashCleanup(phash); //} // -TEST(testCase, create_topic_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_topic_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == nullptr); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// char* sql = "select * from tu"; +// pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); +// taos_free_result(pRes); +// taos_close(pConn); +//} //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -566,58 +567,58 @@ TEST(testCase, create_topic_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// -//TEST(testCase, projection_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -//// pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// pRes = taos_query(pConn, "create table tu using st1 tags(1)"); -//// if (taos_errno(pRes) != 0) { -//// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -//// } -//// taos_free_result(pRes); -//// -//// for(int32_t i = 0; i < 100; ++i) { -//// char sql[512] = {0}; -//// sprintf(sql, "insert into tu values(now+%da, %d)", i, i); -//// TAOS_RES* p = taos_query(pConn, sql); -//// if (taos_errno(p) != 0) { -//// printf("failed to insert data, reason:%s\n", taos_errstr(p)); -//// } -//// -//// taos_free_result(p); -//// } -// -// pRes = taos_query(pConn, "select * from tu"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, projection_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 100000; ++i) { + char sql[512] = {0}; + sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } + + pRes = taos_query(pConn, "select * from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_NE(pConn, nullptr); @@ -678,4 +679,6 @@ TEST(testCase, create_topic_Test) { // // taos_free_result(pRes); // taos_close(pConn); -//} \ No newline at end of file +//} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 40e278f54a..7a0966df57 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -44,7 +44,7 @@ int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; -int32_t tsMaxConnections = 5000; +int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 5c6b9ff6f3..f9755a43b9 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -100,12 +100,12 @@ int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t t } else if (checkTzPresent(timestr, len)) { return parseTimeWithTz(timestr, time, timePrec, 0); } else { - return (*parseLocaltimeFp[day_light])(timestr, time, timePrec); + return (*parseLocaltimeFp[day_light])((char*)timestr, time, timePrec); } } -bool checkTzPresent(const char *str, int32_t len) { - char *seg = forwardToTimeStringEnd(str); +bool checkTzPresent(const char* str, int32_t len) { + char* seg = forwardToTimeStringEnd((char*)str); int32_t seg_len = len - (int32_t)(seg - str); char *c = &seg[seg_len - 1]; @@ -267,7 +267,7 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch #endif int64_t fraction = 0; - str = forwardToTimeStringEnd(timestr); + str = forwardToTimeStringEnd((char*)timestr); if ((str[0] == 'Z' || str[0] == 'z') && str[1] == '\0') { /* utc time, no millisecond, return directly*/ diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 2f821ee8b9..b91b6b06f2 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,7 +1,8 @@ #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wunused-but-set-variable" @@ -94,3 +95,5 @@ TEST(testCase, toInteger_test) { ret = toInteger(s, strlen(s), 10, &val, &sign); ASSERT_EQ(ret, -1); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 32730fc1e3..f952f69f20 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -621,5 +621,7 @@ int32_t dndGetUserAuthFromMnode(SDnode *pDnode, char *user, char *spi, char *enc int32_t code = mndRetriveAuth(pMnode, user, spi, encrypt, secret, ckey); dndReleaseMnode(pDnode, pMnode); + + dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); return code; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index f4fda75bd8..79464347f0 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -171,7 +171,7 @@ static int32_t dndInitClient(SDnode *pDnode) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = "DND-C"; + rpcInit.label = "D-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; @@ -179,9 +179,13 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; - rpcInit.secret = INTERNAL_SECRET; + rpcInit.spi = 1; rpcInit.parent = pDnode; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + rpcInit.secret = pass; + pMgmt->clientRpc = rpcOpen(&rpcInit); if (pMgmt->clientRpc == NULL) { dError("failed to init rpc client"); @@ -256,20 +260,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (strcmp(user, INTERNAL_USER) == 0) { - // A simple temporary implementation - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; return 0; } else if (strcmp(user, TSDB_NETTEST_USER) == 0) { - // A simple temporary implementation - char pass[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass); memcpy(secret, pass, TSDB_PASSWORD_LEN); - *spi = 0; + *spi = 1; *encrypt = 0; *ckey = 0; return 0; @@ -282,12 +284,12 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SDnode *pDnode = parent; if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) { - // dTrace("get internal auth success"); + dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) { - // dTrace("get auth from internal mnode"); + dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } @@ -296,13 +298,12 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char return -1; } - // dDebug("user:%s, send auth msg to other mnodes", user); - SAuthReq *pReq = rpcMallocCont(sizeof(SAuthReq)); tstrncpy(pReq->user, user, TSDB_USER_LEN); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH}; SRpcMsg rpcRsp = {0}; + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, pReq->spi, pReq->encrypt); dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { @@ -314,7 +315,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); *spi = pRsp->spi; *encrypt = pRsp->encrypt; - dDebug("user:%s, success to get user auth from other mnodes", user); + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, pRsp->spi, pRsp->encrypt); } rpcFreeCont(rpcRsp.pCont); @@ -333,7 +334,7 @@ static int32_t dndInitServer(SDnode *pDnode) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = pDnode->cfg.serverPort; - rpcInit.label = "DND-S"; + rpcInit.label = "D-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; rpcInit.sessions = pDnode->cfg.maxShellConns; diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index fd5dcd77d9..3d61db8268 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -27,8 +27,8 @@ void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; tsem_t* TestClient::GetSem() { return &sem; } bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { - char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); @@ -42,7 +42,7 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint rpcInit.ckey = (char*)"key"; rpcInit.parent = this; rpcInit.secret = (char*)secretEncrypt; - // rpcInit.spi = 1; + rpcInit.spi = 1; clientRpc = rpcOpen(&rpcInit); ASSERT(clientRpc); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index aaedf280b5..1507e2a30d 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -328,7 +328,7 @@ typedef struct SMqTopicConsumer { typedef struct SMqConsumerEp { int32_t vgId; // -1 for unassigned - SEpSet epset; + SEpSet epSet; int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; @@ -339,7 +339,7 @@ typedef struct SMqConsumerEp { static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); - tlen += taosEncodeSEpSet(buf, &pConsumerEp->epset); + tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); return tlen; @@ -347,7 +347,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); - buf = taosDecodeSEpSet(buf, &pConsumerEp->epset); + buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 52730412e9..44a5836bd6 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAuth.h" +#include "mndUser.h" static int32_t mndProcessAuthReq(SMnodeMsg *pReq); @@ -25,9 +26,34 @@ int32_t mndInitAuth(SMnode *pMnode) { void mndCleanupAuth(SMnode *pMnode) {} -int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } +int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + SUserObj *pUser = mndAcquireUser(pMnode, user); + if (pUser == NULL) { + *secret = 0; + mError("user:%s, failed to auth user since %s", user, terrstr()); + return -1; + } + + *spi = 1; + *encrypt = 0; + *ckey = 0; + + memcpy(secret, pUser->pass, TSDB_PASSWORD_LEN); + mndReleaseUser(pMnode, pUser); + + mDebug("user:%s, auth info is returned", user); + return 0; +} static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { - mDebug("user:%s, auth req is processed", pReq->user); - return 0; + SAuthReq *pAuth = pReq->rpcMsg.pCont; + + int32_t contLen = sizeof(SAuthRsp); + SAuthRsp *pRsp = rpcMallocCont(contLen); + pReq->pCont = pRsp; + pReq->contLen = contLen; + + int32_t code = mndRetriveAuth(pReq->pMnode, pAuth->user, &pRsp->spi, &pRsp->encrypt, pRsp->secret, pRsp->ckey); + mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, pAuth->spi, pAuth->encrypt, pAuth->user); + return code; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a6634a9f01..817ca4f4be 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -117,7 +117,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // persist msg STransAction action = {0}; - action.epSet = pCEp->epset; + action.epSet = pCEp->epSet; action.pCont = reqStr; action.contLen = tlen; action.msgType = TDMT_VND_MQ_SET_CONN; @@ -142,36 +142,25 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { - SMqConsumerEp CEp; - CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; //convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); SArray *pArray; if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - + return -1; } int32_t sz = taosArrayGetSize(pArray); //convert dag to msg for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp CEp; + CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; STaskInfo* pTaskInfo = taosArrayGet(pArray, i); - int32_t vgId = pTaskInfo->addr.nodeId; - SEpSet epSet; - tConvertQueryAddrToEpSet(&epSet, &pTaskInfo->addr); + tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); + CEp.vgId = pTaskInfo->addr.nodeId; + taosArrayPush(unassignedVg, &CEp); } - /*pTopic->physicalPlan;*/ - SVgObj *pVgroup = NULL; - SSdb *pSdb = pMnode->pSdb; - void *pIter = sdbFetch(pSdb, SDB_VGROUP, NULL, (void **)&pVgroup); - while (pIter != NULL) { - if (pVgroup->dbUid == pTopic->dbUid) { - CEp.epset = mndGetVgroupEpset(pMnode, pVgroup); - CEp.vgId = pVgroup->vgId; - taosArrayPush(unassignedVg, &CEp); - } - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - } - return 0; + qDestroyQueryDag(pDag); + return 0; } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 1d4cbf37ce..ac66e7d88b 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -127,7 +127,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); - SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); pTopic->logicalPlan = calloc(len + 1, sizeof(char)); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index ad378953eb..c36e128893 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -60,9 +60,9 @@ void mndCleanupUser(SMnode *pMnode) {} static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; @@ -202,7 +202,7 @@ SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { SSdb *pSdb = pMnode->pSdb; SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); if (pUser == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; + terrno = TSDB_CODE_MND_USER_NOT_EXIST; } return pUser; } @@ -214,9 +214,9 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) { SUserObj userObj = {0}; + taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass); tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; userObj.superUser = 0; @@ -351,8 +351,9 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { SUserObj newUser = {0}; memcpy(&newUser, pUser, sizeof(SUserObj)); - memset(pUser->pass, 0, sizeof(pUser->pass)); - taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)pAlter->pass, strlen(pAlter->pass), pass); + memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN); newUser.updateTime = taosGetTimestampMs(); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index a9267b0ea3..62472c8c8e 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -152,6 +152,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (insertFp != NULL) { code = (*insertFp)(pSdb, pRow->pObj); if (code != 0) { + code = terrno; taosWLockLatch(pLock); taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 75734d2d29..e625c56db1 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries( PUBLIC tfs PUBLIC wal PUBLIC scheduler + PUBLIC executor PUBLIC qworker ) diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index ec71777882..9cc987d731 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -17,6 +17,8 @@ #define _TD_TQ_H_ #include "common.h" +#include "executor.h" +#include "vnode.h" #include "mallocator.h" #include "meta.h" #include "os.h" @@ -81,27 +83,12 @@ typedef struct STqSubscribeReq { int64_t topic[]; } STqSubscribeReq; -typedef struct STqSubscribeRsp { - STqMsgHead head; - int64_t vgId; - char ep[TSDB_EP_LEN]; // TSDB_EP_LEN -} STqSubscribeRsp; - typedef struct STqHeartbeatReq { } STqHeartbeatReq; typedef struct STqHeartbeatRsp { } STqHeartbeatRsp; -typedef struct STqTopicVhandle { - int64_t topicId; - // executor for filter - void* filterExec; - // callback for mnode - // trigger when vnode list associated topic change - void* (*mCallback)(void*, void*); -} STqTopicVhandle; - #define TQ_BUFFER_SIZE 8 typedef struct STqExec { @@ -162,10 +149,10 @@ typedef struct STqGroup { } STqGroup; typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - SSubQueryMsg* pMsg; + int8_t status; + int64_t offset; + void* dst; + qTaskInfo_t task; } STqTaskItem; // new version @@ -198,10 +185,6 @@ typedef struct STqQueryMsg { struct STqQueryMsg* next; } STqQueryMsg; -typedef struct STqCfg { - // TODO -} STqCfg; - typedef struct STqMemRef { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; @@ -298,6 +281,7 @@ typedef struct STQ { STqMemRef tqMemRef; STqMetaStore* tqMeta; SWal* pWal; + SMeta* pMeta; } STQ; typedef struct STqMgmt { @@ -312,13 +296,15 @@ int tqInit(); void tqCleanUp(); // open in each vnode -STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); +STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); +int tqSetCursor(STQ*, STqSetCurReq* pMsg); + #if 0 int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); int tqSetCursor(STQ*, STqSetCurReq* pMsg); @@ -334,23 +320,6 @@ int tqSendLaunchQuery(STqMsgItem*, int64_t offset); int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq); -typedef struct STqReadHandle { - int64_t ver; - SSubmitMsg* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - SMeta* pMeta; - SArray* pColumnIdList; -} STqReadHandle; - -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList); -void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle* pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); -// return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 499972f476..9e6ecb6e23 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -22,7 +22,6 @@ #include "meta.h" #include "tarray.h" #include "tfs.h" -#include "tq.h" #include "tsdb.h" #include "wal.h" @@ -35,6 +34,12 @@ typedef struct SVnode SVnode; typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef struct STqCfg { + // TODO + int32_t reserved; +} STqCfg; + + typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; @@ -61,6 +66,17 @@ typedef struct { PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeOpt; +typedef struct STqReadHandle { + int64_t ver; + uint64_t tbUid; + SSubmitMsg* pMsg; + SSubmitBlk* pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; + SMeta* pMeta; + SArray* pColIdList; +} STqReadHandle; + /* ------------------------ SVnode ------------------------ */ /** * @brief Initialize the vnode module @@ -180,6 +196,25 @@ int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); +/* ------------------------- TQ QUERY -------------------------- */ + +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta); + +static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { + pReadHandle->pColIdList = pColIdList; +} + +static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t tbUid) { + pHandle->tbUid = tbUid; +} + +void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); +bool tqNextDataBlock(STqReadHandle* pHandle); +int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); +// return SArray +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); + + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index b4e1f57384..2b4200fce5 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -17,6 +17,7 @@ #define _TD_TQ_INT_H_ #include "tq.h" +#include "meta.h" #include "tlog.h" #include "trpc.h" #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index be32ed6829..1fa65b2a73 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -24,6 +24,7 @@ #include "tlockfree.h" #include "tmacro.h" #include "wal.h" +#include "tq.h" #include "vnode.h" diff --git a/source/dnode/vnode/src/meta/metaTbCfg.c b/source/dnode/vnode/src/meta/metaTbCfg.c index 4e02b64ce0..d71d319a79 100644 --- a/source/dnode/vnode/src/meta/metaTbCfg.c +++ b/source/dnode/vnode/src/meta/metaTbCfg.c @@ -32,7 +32,7 @@ size_t metaEncodeTbObjFromTbOptions(const STbCfg *pTbOptions, void *pBuf, size_t switch (pTbOptions->type) { case META_SUPER_TABLE: tlen += taosEncodeFixedU64(ppBuf, pTbOptions->stbCfg.suid); - tlen += tdEncodeSchema(ppBuf, pTbOptions->stbCfg.pTagSchema); + tlen += tdEncodeSchema(ppBuf, (STSchema *)pTbOptions->stbCfg.pTagSchema); // TODO: encode schema version array break; case META_CHILD_TABLE: diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c34c64214d..b18f50cd3f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "../../../../../include/libs/executor/executor.h" #include "tqInt.h" #include "tqMetaStore.h" +#include "tcompare.h" // static // read next version data @@ -51,7 +51,7 @@ void tqCleanUp() { taosTmrCleanUp(tqMgmt.timer); } -STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { +STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -59,6 +59,8 @@ STQ* tqOpen(const char* path, SWal* pWal, STqCfg* tqConfig, SMemAllocatorFactory } pTq->path = strdup(path); pTq->tqConfig = tqConfig; + pTq->pWal = pWal; + pTq->pMeta = pMeta; #if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); @@ -423,7 +425,7 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ /*if (code < 0) {*/ - // TODO: error + // TODO: error /*}*/ // get msgType // if submitblk @@ -608,54 +610,92 @@ int tqItemSSize() { } int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { - SMqCVConsumeReq* pReq = pMsg->pCont; + SMqConsumeReq* pReq = pMsg->pCont; + SRpcMsg rpcMsg; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; - int64_t offset = pReq->offset; + int64_t reqOffset = pReq->offset; + int64_t fetchOffset = reqOffset; int64_t blockingTime = pReq->blockingTime; + int rspLen = 0; + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - int sz = taosArrayGetSize(pConsumer->topics); + int sz = taosArrayGetSize(pConsumer->topics); - for (int i = 0 ; i < sz; i++) { - STqTopicHandle *pHandle = taosArrayGet(pConsumer->topics, i); + for (int i = 0; i < sz; i++) { + STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); - int8_t pos = offset % TQ_BUFFER_SIZE; - int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); - if (old == 1) { - // do nothing + int8_t pos; + int8_t skip = 0; + SWalHead* pHead; + while (1) { + pos = fetchOffset % TQ_BUFFER_SIZE; + skip = atomic_val_compare_exchange_8(&pTopic->buffer.output[pos].status, 0, 1); + if (skip == 1) { + // do nothing + break; + } + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + // check err + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + skip = 1; + break; + } + // read until find TDMT_VND_SUBMIT + pHead = pTopic->pReadhandle->pHead; + if (pHead->head.msgType == TDMT_VND_SUBMIT) { + break; + } + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + skip = 1; + break; + } + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + fetchOffset++; + } + if (skip == 1) continue; + SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; + qTaskInfo_t task = pTopic->buffer.output[pos].task; + + qSetStreamInput(task, pCont); + + //SArray + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + while (1) { + SSDataBlock* pDataBlock; + uint64_t ts; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + break; + } + if (pDataBlock != NULL) { + taosArrayPush(pRes, pDataBlock); + } else { + break; + } + } + + atomic_store_8(&pTopic->buffer.output[pos].status, 0); + + if (taosArrayGetSize(pRes) == 0) { + taosArrayDestroy(pRes); + fetchOffset++; continue; } - if (walReadWithHandle(pHandle->pReadhandle, offset) < 0) { - // TODO - } - SWalHead* pHead = pHandle->pReadhandle->pHead; - while (pHead->head.msgType != TDMT_VND_SUBMIT) { - // read until find TDMT_VND_SUBMIT - } - SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; - SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; - - // TODO: launch query and get output data - void* outputData; - pHandle->buffer.output[pos].dst = outputData; - if (pHandle->buffer.firstOffset == -1 - || pReq->offset < pHandle->buffer.firstOffset) { - pHandle->buffer.firstOffset = pReq->offset; + pTopic->buffer.output[pos].dst = pRes; + if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) { + pTopic->buffer.firstOffset = pReq->offset; } - if (pHandle->buffer.lastOffset == -1 - || pReq->offset > pHandle->buffer.lastOffset) { - pHandle->buffer.lastOffset = pReq->offset; + if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) { + pTopic->buffer.lastOffset = pReq->offset; } - atomic_store_8(&pHandle->buffer.output[pos].status, 1); - // put output into rsp } // launch query // get result - SMqCvConsumeRsp* pRsp; return 0; } @@ -664,40 +704,33 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { if (pConsumer == NULL) { return -1; } - + STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); if (pTopic == NULL) { free(pConsumer); return -1; } - strcpy(pTopic->topicName, pReq->topicName); - strcpy(pTopic->cgroup, pReq->cgroup); + strcpy(pTopic->topicName, pReq->topicName); + strcpy(pTopic->cgroup, pReq->cgroup); strcpy(pTopic->sql, pReq->sql); strcpy(pTopic->logicalPlan, pReq->logicalPlan); strcpy(pTopic->physicalPlan, pReq->physicalPlan); - SArray *pArray; - //TODO: deserialize to SQueryDag - SQueryDag *pDag; - // convert to task - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - // TODO: handle error - } - STaskInfo *pInfo = taosArrayGet(pArray, 0); - SArray* pTasks; - schedulerCopyTask(pInfo, &pTasks, TQ_BUFFER_SIZE); + pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - SSubQueryMsg* pMsg = taosArrayGet(pTasks, i); - pTopic->buffer.output[i].pMsg = pMsg; - pTopic->buffer.output[i].status = 0; - } pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); + if (pTopic->pReadhandle == NULL) { + } + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { + pTopic->buffer.output[i].status = 0; + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, pReadHandle); + } // write mq meta return 0; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) { +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); if (pReadHandle == NULL) { return NULL; @@ -705,7 +738,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SArray* pColumnIdList) { pReadHandle->pMeta = pMeta; pReadHandle->pMsg = NULL; pReadHandle->ver = -1; - pReadHandle->pColumnIdList = pColumnIdList; + pReadHandle->pColIdList = NULL; return NULL; } @@ -717,20 +750,18 @@ void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ve } bool tqNextDataBlock(STqReadHandle* pHandle) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; + while (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) >= 0) { + if (pHandle->tbUid == pHandle->pBlock->uid) return true; } - return true; + return false; } int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - SMemRow row; - int32_t sversion = pHandle->pBlock->sversion; - SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false); - pBlockInfo->numOfCols = pSchema->nCols; + /*int32_t sversion = pHandle->pBlock->sversion;*/ + /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ + pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); pBlockInfo->rows = pHandle->pBlock->numOfRows; pBlockInfo->uid = pHandle->pBlock->uid; - // TODO: filter out unused column return 0; } @@ -763,32 +794,3 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { taosArrayPush(pArray, &colInfo); return pArray; } - -static qTaskInfo_t createExecTaskInfo(SSubQueryMsg *pMsg, void* pStreamBlockReadHandle) { - if (pMsg == NULL || pStreamBlockReadHandle == NULL) { - return NULL; - } - - // print those info into log - pMsg->sId = be64toh(pMsg->sId); - pMsg->queryId = be64toh(pMsg->queryId); - pMsg->taskId = be64toh(pMsg->taskId); - pMsg->contentLen = ntohl(pMsg->contentLen); - - struct SSubplan *plan = NULL; - int32_t code = qStringToSubplan(pMsg->msg, &plan); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } - - qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(pStreamBlockReadHandle, 0, plan, &pTaskInfo, NULL); - if (code != TSDB_CODE_SUCCESS) { - // TODO: destroy SSubplan & pTaskInfo - terrno = code; - return NULL; - } - - return pTaskInfo; -} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index c4bbd93eda..6bbf3b959d 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -127,7 +127,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, pVnode->pWal, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); + pVnode->pTq = tqOpen(dir, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index dd1e5ba9ae..4012914723 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -19,7 +19,10 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); } +int vnodeQueryOpen(SVnode *pVnode) { + return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, + (putReqToQueryQFp)vnodePutReqToVQueryQ); +} int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("query message is processing"); @@ -153,6 +156,11 @@ _exit: return 0; } +static void freeItemHelper(void* pItem) { + char* p = *(char**)pItem; + free(p); +} + /** * @param pVnode * @param pMsg @@ -166,17 +174,20 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { int32_t totalLen = 0; int32_t numOfTables = 0; while ((name = metaTbCursorNext(pCur)) != NULL) { - if (numOfTables < 1000) { // TODO: temp get tables of vnode, and should del when show tables commad ok. + if (numOfTables < 10000) { // TODO: temp get tables of vnode, and should del when show tables commad ok. taosArrayPush(pArray, &name); totalLen += strlen(name); + } else { + tfree(name); } + numOfTables++; } // TODO: temp debug, and should del when show tables command ok - vError("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); - if (numOfTables > 1000) { - numOfTables = 1000; + vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); + if (numOfTables > 10000) { + numOfTables = 10000; } metaCloseTbCursor(pCur); @@ -211,6 +222,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); - taosArrayDestroy(pArray); + taosArrayDestroyEx(pArray, freeItemHelper); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index d1b529f7fb..a6c3f25c6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,8 +83,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error + vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name); } - vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); free(pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { free(pCreateTbReq->stbCfg.pSchema); @@ -95,6 +95,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { free(pCreateTbReq->ntbCfg.pSchema); } } + + vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); break; @@ -129,6 +131,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } + return 0; } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 436593f9d6..a01c3bcf5d 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -16,27 +16,28 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#include "os.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "os.h" +#pragma GCC diagnostic ignored "-Wformat" +#include "addr_any.h" +#include "catalog.h" +#include "stub.h" #include "taos.h" #include "tdef.h" -#include "tvariant.h" -#include "catalog.h" #include "tep.h" #include "trpc.h" -#include "stub.h" -#include "addr_any.h" - - +#include "tvariant.h" namespace { -extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist); +extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, + int32_t *exist); extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output); void ctgTestSetPrepareTableMeta(); @@ -44,9 +45,9 @@ void ctgTestSetPrepareCTableMeta(); void ctgTestSetPrepareSTableMeta(); void ctgTestSetPrepareMultiSTableMeta(); -bool ctgTestStop = false; -bool ctgTestEnableSleep = false; -bool ctgTestDeadLoop = false; +bool ctgTestStop = false; +bool ctgTestEnableSleep = false; +bool ctgTestDeadLoop = false; int32_t ctgTestPrintNum = 200000; int32_t ctgTestMTRunSec = 30; @@ -61,14 +62,13 @@ int32_t ctgTestSuid = 2; int64_t ctgTestDbId = 33; uint64_t ctgTestClusterId = 0x1; -char *ctgTestDbname = "1.db1"; -char *ctgTestTablename = "table1"; -char *ctgTestCTablename = "ctable1"; -char *ctgTestSTablename = "stable1"; - +char *ctgTestDbname = "1.db1"; +char *ctgTestTablename = "table1"; +char *ctgTestCTablename = "ctable1"; +char *ctgTestSTablename = "stable1"; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(sizeof(SCreateDbReq)); + SCreateDbReq *pReq = (SCreateDbReq *)rpcMallocCont(sizeof(SCreateDbReq)); strcpy(pReq->db, "1.db1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -89,7 +89,7 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { pReq->update = 0; pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; rpcMsg.contLen = sizeof(SCreateDbReq); @@ -103,8 +103,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { } void ctgTestInitLogFile() { - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; tsAsyncLog = 0; @@ -113,7 +113,6 @@ void ctgTestInitLogFile() { if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } - } int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { @@ -152,10 +151,10 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { output->tbMeta->tableInfo.numOfColumns = ctgTestColNum; output->tbMeta->tableInfo.numOfTags = ctgTestTagNum; - + output->tbMeta->sversion = ctgTestSVersion; output->tbMeta->tversion = ctgTestTVersion; - + SSchema *s = NULL; s = &output->tbMeta->schema[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; @@ -174,18 +173,17 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { s->colId = 3; s->bytes = 12; strcpy(s->name, "tag1s"); - } void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; - int32_t vgNum = 0; - SVgroupInfo vgInfo = {0}; - + int32_t vgNum = 0; + SVgroupInfo vgInfo = {0}; + dbVgroup->vgVersion = vgVersion++; - + ctgTestCurrentVgVersion = dbVgroup->vgVersion; - + dbVgroup->hashMethod = 0; dbVgroup->dbId = ctgTestDbId; dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -210,9 +208,9 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { } void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - SUseDbRsp *rspMsg = NULL; //todo + SUseDbRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(SUseDbRsp) + ctgTestVgNum * sizeof(SVgroupInfo); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (SUseDbRsp *)pRsp->pCont; @@ -224,7 +222,7 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM rspMsg->uid = htobe64(ctgTestDbId); SVgroupInfo *vg = NULL; - uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; for (int32_t i = 0; i < ctgTestVgNum; ++i) { vg = &rspMsg->vgroupInfo[i]; @@ -245,13 +243,10 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM return; } - - - void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -279,15 +274,14 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM s->colId = htonl(2); s->bytes = htonl(4); strcpy(s->name, "col1"); - + return; } - void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -323,15 +317,13 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc s->bytes = htonl(12); strcpy(s->name, "tag1s"); - return; } - void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -367,15 +359,14 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc s->bytes = htonl(12); strcpy(s->name, "tag1s"); - return; } void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; //todo + STableMetaRsp *rspMsg = NULL; // todo static int32_t idx = 1; - pRsp->code =0; + pRsp->code = 0; pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; @@ -412,55 +403,50 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, strcpy(s->name, "tag1s"); ++idx; - + return; } - - void ctgTestPrepareDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareTableMeta(); - + return; } - void ctgTestPrepareDbVgroupsAndChildMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareCTableMeta(); - + return; } void ctgTestPrepareDbVgroupsAndSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareSTableMeta(); - + return; } void ctgTestPrepareDbVgroupsAndMultiSuperMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestPrepareDbVgroups(shandle, pEpSet, pMsg, pRsp); - + ctgTestSetPrepareMultiSTableMeta(); - + return; } - - void ctgTestSetPrepareDbVgroups() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroups); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroups); } } @@ -470,10 +456,10 @@ void ctgTestSetPrepareTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareTableMeta); } } @@ -483,10 +469,10 @@ void ctgTestSetPrepareCTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareCTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareCTableMeta); } } @@ -496,10 +482,10 @@ void ctgTestSetPrepareSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareSTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareSTableMeta); } } @@ -509,38 +495,36 @@ void ctgTestSetPrepareMultiSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareMultiSTableMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareMultiSTableMeta); } } } - void ctgTestSetPrepareDbVgroupsAndNormalMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndNormalMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndNormalMeta); } } } - void ctgTestSetPrepareDbVgroupsAndChildMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndChildMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndChildMeta); } } @@ -550,10 +534,10 @@ void ctgTestSetPrepareDbVgroupsAndSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndSuperMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndSuperMeta); } } @@ -563,25 +547,24 @@ void ctgTestSetPrepareDbVgroupsAndMultiSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestPrepareDbVgroupsAndMultiSuperMeta); { - AddrAny any("libtransport.so"); - std::map result; + AddrAny any("libtransport.so"); + std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); - for (const auto& f : result) { + for (const auto &f : result) { stub.set(f.second, ctgTestPrepareDbVgroupsAndMultiSuperMeta); } } } - -} +} // namespace void *ctgTestGetDbVgroupThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - void *mockPointer = (void *)0x1; - SArray *vgList = NULL; - int32_t n = 0; - + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + void *mockPointer = (void *)0x1; + SArray *vgList = NULL; + int32_t n = 0; + while (!ctgTestStop) { code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); if (code) { @@ -593,7 +576,7 @@ void *ctgTestGetDbVgroupThread(void *param) { } if (ctgTestEnableSleep) { - usleep(rand()%5); + usleep(rand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); @@ -604,11 +587,11 @@ void *ctgTestGetDbVgroupThread(void *param) { } void *ctgTestSetDbVgroupThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; - int32_t n = 0; - + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + SDBVgroupInfo dbVgroup = {0}; + int32_t n = 0; + while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); @@ -616,8 +599,8 @@ void *ctgTestSetDbVgroupThread(void *param) { assert(0); } - if (ctgTestEnableSleep) { - usleep(rand()%5); + if (ctgTestEnableSleep) { + usleep(rand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -625,20 +608,19 @@ void *ctgTestSetDbVgroupThread(void *param) { } return NULL; - } void *ctgTestGetCtableMetaThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - int32_t n = 0; - STableMeta* tbMeta = NULL; - int32_t exist = 0; + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + int32_t n = 0; + STableMeta *tbMeta = NULL; + int32_t exist = 0; SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); - + while (!ctgTestStop) { code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist); if (code || 0 == exist) { @@ -648,9 +630,9 @@ void *ctgTestGetCtableMetaThread(void *param) { tfree(tbMeta); if (ctgTestEnableSleep) { - usleep(rand()%5); + usleep(rand() % 5); } - + if (++n % ctgTestPrintNum == 0) { printf("Get:%d\n", n); } @@ -660,22 +642,22 @@ void *ctgTestGetCtableMetaThread(void *param) { } void *ctgTestSetCtableMetaThread(void *param) { - struct SCatalog* pCtg = (struct SCatalog*)param; - int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; - int32_t n = 0; + struct SCatalog *pCtg = (struct SCatalog *)param; + int32_t code = 0; + SDBVgroupInfo dbVgroup = {0}; + int32_t n = 0; STableMetaOutput output = {0}; ctgTestBuildCTableMetaOutput(&output); - + while (!ctgTestStop) { code = ctgUpdateTableMetaCache(pCtg, &output); if (code) { assert(0); } - if (ctgTestEnableSleep) { - usleep(rand()%5); + if (ctgTestEnableSleep) { + usleep(rand() % 5); } if (++n % ctgTestPrintNum == 0) { printf("Set:%d\n", n); @@ -685,21 +667,19 @@ void *ctgTestSetCtableMetaThread(void *param) { tfree(output.tbMeta); return NULL; - } - TEST(tableMeta, normalTable) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; ctgTestSetPrepareDbVgroups(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -715,7 +695,6 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(vgInfo.vgId, 8); ASSERT_EQ(vgInfo.numOfEps, 3); - ctgTestSetPrepareTableMeta(); STableMeta *tableMeta = NULL; @@ -742,19 +721,19 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stb = NULL; - uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; - int32_t i = 0; + uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; + int32_t i = 0; while (i < 5) { ++i; code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum); ASSERT_EQ(code, 0); code = catalogGetExpiredSTables(pCtg, &stb, &stbNum); ASSERT_EQ(code, 0); - + if (dbNum) { - printf("got expired db,dbId:%"PRId64"\n", dbs->dbId); + printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); free(dbs); dbs = NULL; } else { @@ -762,7 +741,7 @@ TEST(tableMeta, normalTable) { } if (stbNum) { - printf("got expired stb,suid:%"PRId64"\n", stb->suid); + printf("got expired stb,suid:%" PRId64 "\n", stb->suid); free(stb); stb = NULL; } else { @@ -773,7 +752,7 @@ TEST(tableMeta, normalTable) { allStbNum += stbNum; sleep(2); } - + ASSERT_EQ(allDbNum, 1); ASSERT_EQ(allStbNum, 0); @@ -781,18 +760,18 @@ TEST(tableMeta, normalTable) { } TEST(tableMeta, childTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - + code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -838,19 +817,19 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stb = NULL; - uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; - int32_t i = 0; + uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; + int32_t i = 0; while (i < 5) { ++i; code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum); ASSERT_EQ(code, 0); code = catalogGetExpiredSTables(pCtg, &stb, &stbNum); ASSERT_EQ(code, 0); - + if (dbNum) { - printf("got expired db,dbId:%"PRId64"\n", dbs->dbId); + printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); free(dbs); dbs = NULL; } else { @@ -858,7 +837,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%"PRId64"\n", stb->suid); + printf("got expired stb,suid:%" PRId64 "\n", stb->suid); free(stb); stb = NULL; } else { @@ -869,18 +848,17 @@ TEST(tableMeta, childTableCase) { allStbNum += stbNum; sleep(2); } - + ASSERT_EQ(allDbNum, 1); ASSERT_EQ(allStbNum, 1); - catalogDestroy(); } TEST(tableMeta, superTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; ctgTestSetPrepareDbVgroupsAndSuperMeta(); @@ -889,7 +867,7 @@ TEST(tableMeta, superTableCase) { int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -940,19 +918,19 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stb = NULL; - uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; - int32_t i = 0; + uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; + int32_t i = 0; while (i < 5) { ++i; code = catalogGetExpiredDBs(pCtg, &dbs, &dbNum); ASSERT_EQ(code, 0); code = catalogGetExpiredSTables(pCtg, &stb, &stbNum); ASSERT_EQ(code, 0); - + if (dbNum) { - printf("got expired db,dbId:%"PRId64"\n", dbs->dbId); + printf("got expired db,dbId:%" PRId64 "\n", dbs->dbId); free(dbs); dbs = NULL; } else { @@ -960,7 +938,7 @@ TEST(tableMeta, superTableCase) { } if (stbNum) { - printf("got expired stb,suid:%"PRId64"\n", stb->suid); + printf("got expired stb,suid:%" PRId64 "\n", stb->suid); free(stb); stb = NULL; } else { @@ -971,19 +949,18 @@ TEST(tableMeta, superTableCase) { allStbNum += stbNum; sleep(2); } - + ASSERT_EQ(allDbNum, 1); ASSERT_EQ(allStbNum, 1); - catalogDestroy(); } TEST(tableDistVgroup, normalTable) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndNormalMeta(); @@ -992,8 +969,8 @@ TEST(tableDistVgroup, normalTable) { int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -1012,17 +989,17 @@ TEST(tableDistVgroup, normalTable) { } TEST(tableDistVgroup, childTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1040,15 +1017,14 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(vgInfo->vgId, 9); ASSERT_EQ(vgInfo->numOfEps, 4); - catalogDestroy(); } TEST(tableDistVgroup, superTableCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo *vgInfo = NULL; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo *vgInfo = NULL; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndSuperMeta(); @@ -1057,7 +1033,7 @@ TEST(tableDistVgroup, superTableCase) { int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); @@ -1078,24 +1054,23 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(vgInfo->vgId, 3); ASSERT_EQ(vgInfo->numOfEps, 3); - catalogDestroy(); } TEST(dbVgroup, getSetDbVgroupCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestSetPrepareDbVgroupsAndNormalMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1109,7 +1084,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetDBVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, ctgTestDbname, false, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), ctgTestVgNum); - + code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); @@ -1139,27 +1114,27 @@ TEST(dbVgroup, getSetDbVgroupCase) { ASSERT_EQ(pvgInfo->vgId, 8); ASSERT_EQ(pvgInfo->numOfEps, 3); taosArrayDestroy(vgList); - + catalogDestroy(); } TEST(multiThread, getSetDbVgroupCase) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestStop = false; ctgTestInitLogFile(); - + ctgTestSetPrepareDbVgroups(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1187,28 +1162,28 @@ TEST(multiThread, getSetDbVgroupCase) { break; } } - + ctgTestStop = true; sleep(1); - + catalogDestroy(); } TEST(multiThread, ctableMeta) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestStop = false; ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); - //sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); - + // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + int32_t code = catalogInit(NULL); ASSERT_EQ(code, 0); @@ -1235,25 +1210,24 @@ TEST(multiThread, ctableMeta) { break; } } - + ctgTestStop = true; sleep(1); - + catalogDestroy(); } - TEST(rentTest, allRent) { - struct SCatalog* pCtg = NULL; - void *mockPointer = (void *)0x1; - SVgroupInfo vgInfo = {0}; - SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; - SArray *vgList = NULL; + struct SCatalog *pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + SVgroupInfo *pvgInfo = NULL; + SDBVgroupInfo dbVgroup = {0}; + SArray *vgList = NULL; ctgTestStop = false; - SDbVgVersion *dbs = NULL; + SDbVgVersion *dbs = NULL; SSTableMetaVersion *stable = NULL; - uint32_t num = 0; + uint32_t num = 0; ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); @@ -1265,7 +1239,6 @@ TEST(rentTest, allRent) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; strcpy(n.dbname, "db1"); @@ -1285,41 +1258,37 @@ TEST(rentTest, allRent) { ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum); ASSERT_EQ(tableMeta->tableInfo.precision, 1); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); - + code = catalogGetExpiredDBs(pCtg, &dbs, &num); ASSERT_EQ(code, 0); printf("%d - expired dbNum:%d\n", i, num); if (dbs) { - printf("%d - expired dbId:%"PRId64", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); + printf("%d - expired dbId:%" PRId64 ", vgVersion:%d\n", i, dbs->dbId, dbs->vgVersion); free(dbs); dbs = NULL; } - + code = catalogGetExpiredSTables(pCtg, &stable, &num); ASSERT_EQ(code, 0); printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%"PRId64", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; } printf("*************************************************\n"); - + sleep(2); } - + catalogDestroy(); } - - -int main(int argc, char** argv) { +int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } - - - \ No newline at end of file +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 73a30a62f5..ebf3d83a1a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -372,11 +372,14 @@ typedef struct STaskParam { typedef struct SExchangeInfo { SArray *pSources; - uint64_t bytes; // total load bytes from remote tsem_t ready; void *pTransporter; SRetrieveTableRsp *pRsp; SSDataBlock *pResult; + int32_t current; + uint64_t rowsOfCurrentSource; + uint64_t bytes; // total load bytes from remote + uint64_t totalRows; } SExchangeInfo; typedef struct STableScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6dea4a4e57..2f1f40813c 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -11,4 +11,82 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "executor.h" +#include "tq.h" +#include "executorimpl.h" +#include "planner.h" + +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t reqId) { + ASSERT(pOperator != NULL); + if (pOperator->operatorType != OP_StreamScan) { + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator to set the input data block, reqId:0x%" PRIx64, reqId); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (pOperator->numOfDownstream > 1) { // not handle this in join query + qError("join not supported for stream block scan, reqId:0x%" PRIx64, reqId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return doSetStreamBlock(pOperator->pDownstream[0], input, reqId); + } else { + SStreamBlockScanInfo* pInfo = pOperator->info; + tqReadHandleSetMsg(pInfo->readerHandle, input, 0); + return TSDB_CODE_SUCCESS; + } +} + +int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { + if (tinfo == NULL) { + return TSDB_CODE_QRY_APP_ERROR; + } + + if (input == NULL) { + return TSDB_CODE_SUCCESS; + } + + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; + + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to set the stream block data, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); + } else { + qDebug("set the stream block successfully, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); + } + + return code; +} + +qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle) { + if (pMsg == NULL || streamReadHandle == NULL) { + return NULL; + } + + // print those info into log +#if 0 + pMsg->sId = pMsg->sId; + pMsg->queryId = pMsg->queryId; + pMsg->taskId = pMsg->taskId; + pMsg->contentLen = pMsg->contentLen; +#endif + + struct SSubplan* plan = NULL; + int32_t code = qStringToSubplan(pMsg->msg, &plan); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(streamReadHandle, 0, plan, &pTaskInfo, NULL); + if (code != TSDB_CODE_SUCCESS) { + // TODO: destroy SSubplan & pTaskInfo + terrno = code; + return NULL; + } + + return pTaskInfo; +} diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index c5b57ee1a5..e5d56aca15 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -164,13 +164,6 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { return TSDB_CODE_SUCCESS; } - // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; - // if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { - // qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); - // setTaskStatus(pTaskInfo, TASK_COMPLETED); - // return doBuildResCheck(pTaskInfo); - // } - // error occurs, record the error code and return to client int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0ed480ed15..2645b3eac8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -27,6 +27,7 @@ #include "thash.h" #include "ttypes.h" #include "query.h" +#include "vnode.h" #include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) @@ -4957,6 +4958,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + if (pTableScanInfo->pTsdbReadHandle == NULL) { + return NULL; + } + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -5143,72 +5148,111 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; *newgroup = false; - if (pExchangeInfo->pRsp != NULL && pExchangeInfo->pRsp->completed == 1) { + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + if (pExchangeInfo->current >= totalSources) { return NULL; } - SResFetchReq *pMsg = calloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { // todo handle malloc error - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _error; - } + SResFetchReq* pMsg = NULL; + SMsgSendInfo* pMsgSendInfo = NULL; - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, 0); - SEpSet epSet = {0}; - - epSet.numOfEps = pSource->addr.numOfEps; - epSet.port[0] = pSource->addr.epAddr[0].port; - tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); - - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("QID:%"PRIx64" calloc %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _error; - } - - pMsgSendInfo->param = pExchangeInfo; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = TDMT_VND_FETCH; - pMsgSendInfo->fp = loadRemoteDataCallback; - - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pExchangeInfo->ready); - - if (pExchangeInfo->pRsp->numOfRows == 0) { - return NULL; - } - - SSDataBlock* pRes = pExchangeInfo->pResult; - char* pData = pExchangeInfo->pRsp->data; - - for(int32_t i = 0; i < pOperator->numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); - char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pExchangeInfo->pRsp->numOfRows); - if (tmp == NULL) { + while(1) { + pMsg = calloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { // todo handle malloc error + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _error; } - size_t len = pExchangeInfo->pRsp->numOfRows * pColInfoData->info.bytes; - memcpy(tmp, pData, len); + SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - pColInfoData->pData = tmp; - pData += len; + SEpSet epSet = {0}; + epSet.numOfEps = pSource->addr.numOfEps; + epSet.port[0] = pSource->addr.epAddr[0].port; + tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); + + qDebug("QID:0x%" PRIx64 " build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + + // send the fetch remote task result reques + pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("QID:0x%" PRIx64 " prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _error; + } + + pMsgSendInfo->param = pExchangeInfo; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = TDMT_VND_FETCH; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pExchangeInfo->ready); + + SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; + if (pRsp->numOfRows == 0) { + qDebug("QID:0x%"PRIx64" vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, + pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows); + + pExchangeInfo->rowsOfCurrentSource = 0; + pExchangeInfo->current += 1; + + if (pExchangeInfo->current >= totalSources) { + return NULL; + } else { + continue; + } + } + + SSDataBlock* pRes = pExchangeInfo->pResult; + char* pData = pRsp->data; + + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); + char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows); + if (tmp == NULL) { + goto _error; + } + + size_t len = pRsp->numOfRows * pColInfoData->info.bytes; + memcpy(tmp, pData, len); + + pColInfoData->pData = tmp; + pData += len; + } + + pRes->info.numOfCols = pOperator->numOfOutput; + pRes->info.rows = pRsp->numOfRows; + + pExchangeInfo->totalRows += pRsp->numOfRows; + pExchangeInfo->bytes += pRsp->compLen; + pExchangeInfo->rowsOfCurrentSource += pRsp->numOfRows; + + if (pRsp->completed == 1) { + qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->bytes, + pExchangeInfo->current + 1, totalSources); + + pExchangeInfo->rowsOfCurrentSource = 0; + pExchangeInfo->current += 1; + } else { + qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->bytes); + } + + return pExchangeInfo->pResult; } - pRes->info.numOfCols = pOperator->numOfOutput; - pRes->info.rows = pExchangeInfo->pRsp->numOfRows; - - return pExchangeInfo->pResult; - _error: tfree(pMsg); tfree(pMsgSendInfo); @@ -5258,7 +5302,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* rpcInit.user = (char *)"root"; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.ckey = "key"; -// rpcInit.spi = 1; + rpcInit.spi = 1; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; pInfo->pTransporter = rpcOpen(&rpcInit); @@ -5407,7 +5451,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt return pOperator; } -SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, uint64_t uid, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5417,16 +5461,29 @@ SOperatorInfo* createStreamBlockScanOperatorInfo(void *pStreamBlockHandle, int32 return NULL; } - pInfo->readerHandle = pStreamBlockHandle; + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo); + SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + + taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId); + } + + // set the extract column id to streamHandle + tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); + tqReadHandleSetTbUid(streamReadHandle, uid); + + pInfo->readerHandle = streamReadHandle; pOperator->name = "StreamBlockScanOperator"; - pOperator->operatorType = OP_StreamBlockScan; + pOperator->operatorType = OP_StreamScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; pOperator->exec = doStreamBlockScan; pOperator->pTaskInfo = pTaskInfo; + return pOperator; } @@ -7704,6 +7761,9 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); + } else if (pPhyNode->info.type == OP_StreamScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. + return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo); } } @@ -7768,11 +7828,12 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, if (groupInfo.numOfTables == 0) { code = 0; - // qDebug("no table qualified for query, reqId:0x%"PRIx64, (*pTask)->id.queryId); + qDebug("no table qualified for query, reqId:0x%"PRIx64, queryId); goto _error; } return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId); + _error: terrno = code; return NULL; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index c528d879a3..5e0b641b28 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -17,8 +17,9 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -219,4 +220,6 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo, &sinkHandle); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 5852678880..06a9a3d16e 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -26,7 +26,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out const char* msg4 = "pattern is invalid"; const char* msg5 = "database name is empty"; const char* msg6 = "pattern string is empty"; - const char* msg7 = "db is not specified"; + const char* msg7 = "database not specified"; /* * database prefix in pInfo->pMiscInfo->a[0] * wildcard in like clause in pInfo->pMiscInfo->a[1] @@ -50,7 +50,11 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); - catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + int32_t code = catalogGetDBVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbFname, false, &array); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); @@ -62,9 +66,8 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out pEpSet->port[i] = info->epAddr[i].port; } - *outputLen = sizeof(SVShowTablesReq); - *output = pShowReq; - + *outputLen = sizeof(SVShowTablesReq); + *output = pShowReq; *pExtension = array; } else { if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 58e368aa0d..4271aae451 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -36,25 +36,29 @@ bool qIsDdlQuery(const SQueryNode* pQueryNode) { } int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { + int32_t code = TSDB_CODE_SUCCESS; + SSqlInfo info = doGenerateAST(pCxt->pSql); if (!info.valid) { strncpy(pCxt->pMsg, info.msg, pCxt->msgLen); - terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - return terrno; + code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + goto _end; } if (!isDqlSqlStatement(&info)) { if (info.type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pModifStmtInfo == NULL) { - return terrno; + code = terrno; + goto _end; } *pQuery = (SQueryNode*)pModifStmtInfo; } else { SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pDcl == NULL) { - return terrno; + code = terrno; + goto _end; } *pQuery = (SQueryNode*)pDcl; @@ -63,21 +67,22 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { } else { SQueryStmtInfo* pQueryInfo = createQueryInfo(); if (pQueryInfo == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code. - return terrno; + code = TSDB_CODE_QRY_OUT_OF_MEMORY; // set correct error code. + goto _end; } - int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); + code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } else { - terrno = code; - return code; + goto _end; } } + _end: destroySqlInfo(&info); - return TSDB_CODE_SUCCESS; + terrno = code; + return code; } int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQueryNode) { @@ -247,5 +252,6 @@ void qDestroyQuery(SQueryNode* pQueryNode) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); } + tfree(pQueryNode); } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index e8d975c22e..d68b04a384 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -18,8 +18,14 @@ #include #include "stub.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" + #include "addr_any.h" +#pragma GCC diagnostic pop + namespace { void generateTestT1(MockCatalogService* mcs) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 4847b50082..b408e15abe 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -17,8 +17,9 @@ #include #include #include "tglobal.h" -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -781,4 +782,6 @@ TEST(testCase, create_user_Test) { ASSERT_NE(output, nullptr); destroySqlInfo(&info1); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index a278e0053c..43daff9fec 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -17,8 +17,9 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -203,4 +204,6 @@ TEST(testCase, displayPlan) { // Projection(cols: [ts #0], [a #1], [b #2]) filters:(nil) // TableScan(t.1abc #110) time_range: -9223372036854775808 - 9223372036854775807 -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 7ed052e8cc..888eeae73b 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -1,7 +1,8 @@ #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -722,3 +723,4 @@ TEST(testCase, extractMeta_test) { destroySqlInfo(&info1); } +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 41f50607cb..26ae44a08f 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -106,7 +106,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); */ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); -int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId); +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId); void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource); int32_t subPlanToString(const SSubplan *pPhyNode, char** str, int32_t* len); int32_t stringToSubplan(const char* str, SSubplan** subplan); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index dd869d87b3..0038c51c7a 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -261,14 +261,14 @@ static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) { return; } -static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { - SVgroupsInfo* pVgroupList = pTable->pMeta->vgroupList; +static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) { + SVgroupsInfo* pVgroupList = pTableInfo->pMeta->vgroupList; for (int32_t i = 0; i < pVgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); subplan->msgType = TDMT_VND_QUERY; - vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execNode); - subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + vgroupMsgToEpSet(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode); + subplan->pNode = createMultiTableScanNode(pPlanNode, pTableInfo); subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } @@ -384,18 +384,19 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { subplan->msgType = TDMT_VND_QUERY; subplan->pNode = createPhyNode(pCxt, pRoot); - subplan->pDataSink = createDataDispatcher(pCxt, pRoot, subplan->pNode); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot, subplan->pNode); } // todo deal subquery } -int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, uint64_t requestId) { +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId) { TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, - .nextId = {.queryId = requestId}, + //The unsigned Id starting from 1 would be better + .nextId = {.queryId = requestId, .subplanId = 1, .templateId = 1}, }; *pDag = context.pDag; @@ -408,6 +409,17 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD terrno = code; return TSDB_CODE_FAILED; } END_TRY + + // traverse the dag again to acquire the execution node. + if (pNodeList != NULL) { + SArray** pSubLevel = taosArrayGetLast((*pDag)->pSubplans); + size_t num = taosArrayGetSize(*pSubLevel); + for (int32_t j = 0; j < num; ++j) { + SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); + taosArrayPush(pNodeList, &pPlan->execNode); + } + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 9b32213ad7..c93569a6c1 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -16,6 +16,8 @@ #include "parser.h" #include "plannerInt.h" +static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols); + static void destroyDataSinkNode(SDataSink* pSinkNode) { if (pSinkNode == NULL) { return; @@ -56,7 +58,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId) { SQueryPlanNode* pLogicPlan; int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { @@ -76,17 +78,31 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, return code; } - code = createDag(pLogicPlan, NULL, pDag, requestId); + code = createDag(pLogicPlan, NULL, pDag, pNodeList, requestId); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(pLogicPlan); qDestroyQueryDag(*pDag); return code; } + extractResSchema(pDag, pResSchema, numOfCols); + destroyQueryPlan(pLogicPlan); return TSDB_CODE_SUCCESS; } +void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, + int32_t* numOfCols) { // extract the final result schema + SArray* pTopSubplan = taosArrayGetP((*pDag)->pSubplans, 0); + + SSubplan* pPlan = taosArrayGetP(pTopSubplan, 0); + SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); + + *numOfCols = pDataBlockSchema->numOfCols; + *pResSchema = calloc(pDataBlockSchema->numOfCols, sizeof(SSchema)); + memcpy((*pResSchema), pDataBlockSchema->pSchema, pDataBlockSchema->numOfCols * sizeof(SSchema)); +} + void qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { setSubplanExecutionNode(subplan, templateId, pSource); } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 6d9e08e829..edf5fa5a81 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -61,7 +61,7 @@ protected: int32_t run() { SQueryDag* dag = nullptr; uint64_t requestId = 20; - int32_t code = createDag(logicPlan_.get(), nullptr, &dag, requestId); + int32_t code = createDag(logicPlan_.get(), nullptr, &dag, NULL, requestId); dag_.reset(dag); return code; } @@ -78,9 +78,9 @@ protected: SQueryDag* dag = nullptr; uint64_t requestId = 20; SSchema *schema = NULL; - uint32_t numOfOutput = 0; + int32_t numOfOutput = 0; - code = qCreateQueryDag(query, &dag, requestId); + code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, nullptr, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 4b408e67db..765651a31a 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -22,8 +22,8 @@ #include "parser.h" #include "mockCatalog.h" +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" - #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -102,4 +102,6 @@ TEST(testCase, planner_test) { // destroyQueryInfo(pQueryInfo); // qParserCleanupMetaRequestInfo(&req); // destroySqlInfo(&info1); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 2cde28baf9..52e632ffbb 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -18,6 +18,9 @@ #include "query.h" #include "trpc.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-truncation" + int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize) = {0}; @@ -288,7 +291,4 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } - - - - +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp index 8fc6b7e529..9ca7442d55 100644 --- a/source/libs/qcom/test/queryTest.cpp +++ b/source/libs/qcom/test/queryTest.cpp @@ -18,8 +18,9 @@ #include "tmsg.h" #include "query.h" #include "trpc.h" -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -82,3 +83,5 @@ TEST(testCase, error_in_async_test) { usleep(1000); printf("Error code:%d after asynchronously exec function\n", code); } + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 5f9b33f7e3..01c079e1f7 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -171,17 +171,17 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_ELOG_E(param) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) @@ -219,8 +219,7 @@ typedef struct SQWorkerMgmt { } \ } while (0) - - +int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); #ifdef __cplusplus } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index d1cc9f03d1..0ed5c0c816 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -16,11 +16,17 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + #include "os.h" #include "taos.h" @@ -461,11 +467,11 @@ void *controlThread(void *param) { } void *queryQueueThread(void *param) { - + return NULL; } void *fetchQueueThread(void *param) { - + return NULL; } @@ -783,5 +789,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } - - +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 6b047eb96e..2ace1c66f5 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -146,12 +146,15 @@ typedef struct SSchJob { #define SCH_SET_JOB_TYPE(pAttr, type) (pAttr)->queryJob = ((type) != QUERY_TYPE_MODIFY) #define SCH_JOB_NEED_FETCH(pAttr) ((pAttr)->queryJob) -#define SCH_JOB_ELOG(param, ...) qError("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) -#define SCH_JOB_DLOG(param, ...) qDebug("QID:%"PRIx64" " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) +#define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) -#define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) -#define SCH_TASK_WLOG(param, ...) qWarn("QID:%"PRIx64",TID:%"PRId64" " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_ELOG(param, ...) \ + qError("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_DLOG(param, ...) \ + qDebug("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) +#define SCH_TASK_WLOG(param, ...) \ + qWarn("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index b3e827221c..a6b499aca8 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -639,7 +639,6 @@ _return: int32_t schProcessOnDataFetched(SSchJob *job) { atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); - tsem_post(&job->rspSem); } @@ -858,11 +857,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - atomic_store_ptr(&pJob->res, rsp); - atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); - - if (rsp->completed) { - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + SCH_ERR_JRET(schProcessOnDataFetched(pJob)); + break; } SCH_ERR_JRET(schProcessOnDataFetched(pJob)); @@ -876,7 +872,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch } default: SCH_TASK_ELOG("unknown rsp msg, type:%d, status:%d", msgType, SCH_GET_TASK_STATUS(pTask)); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -918,9 +913,8 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in } pTask = *task; - - SCH_TASK_DLOG("rsp msg received, type:%d, code:%x", msgType, rspCode); - + SCH_TASK_DLOG("rsp msg received, type:%s, code:%s", TMSG_INFO(msgType), tstrerror(rspCode)); + SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: @@ -1468,12 +1462,12 @@ int32_t schedulerConvertDagToTaskList(SQueryDag *pDag, SArray **pTasks) { SSubQueryMsg *pMsg = (SSubQueryMsg*) msg; pMsg->header.vgId = htonl(tInfo.addr.nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(plan->id.queryId); - pMsg->taskId = htobe64(schGenUUID()); + + pMsg->sId = schMgmt.sId; + pMsg->queryId = plan->id.queryId; + pMsg->taskId = schGenUUID(); pMsg->taskType = TASK_TYPE_PERSISTENT; - pMsg->contentLen = htonl(msgLen); + pMsg->contentLen = msgLen; memcpy(pMsg->msg, msg, msgLen); tInfo.msg = pMsg; @@ -1545,8 +1539,8 @@ int32_t scheduleFetchRows(SSchJob *pJob, void **pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0; atomic_add_fetch_32(&pJob->ref, 1); int8_t status = SCH_GET_JOB_STATUS(pJob); @@ -1592,7 +1586,6 @@ _return: while (true) { *pData = atomic_load_ptr(&pJob->res); - if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { continue; } @@ -1611,8 +1604,7 @@ _return: atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); - SCH_JOB_DLOG("fetch done, code:%x", code); - + SCH_JOB_DLOG("fetch done, code:%s", tstrerror(code)); atomic_sub_fetch_32(&pJob->ref, 1); SCH_RET(code); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 1425ac0e6c..fb13149713 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -16,11 +16,7 @@ #include #include #include -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" #include "taos.h" @@ -30,6 +26,16 @@ #include "scheduler.h" #include "tep.h" #include "trpc.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wliteral-suffix" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wreturn-type" +#pragma GCC diagnostic ignored "-Wformat" + #include "schedulerInt.h" #include "stub.h" #include "addr_any.h" @@ -680,6 +686,4 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } - - - +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/sync/src/raft_handle_vote_message.c b/source/libs/sync/src/raft_handle_vote_message.c index 0219e39df9..4d940732dc 100644 --- a/source/libs/sync/src/raft_handle_vote_message.c +++ b/source/libs/sync/src/raft_handle_vote_message.c @@ -37,7 +37,7 @@ int syncRaftHandleVoteMessage(SSyncRaft* pRaft, const SSyncMessage* pMsg) { if (pRespMsg == NULL) { return 0; } - syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d"\ + syncInfo("[%d:%d] [logterm: %" PRId64 ", index: %" PRId64 ", vote: %d] %s for %d" "[logterm: %" PRId64 ", index: %" PRId64 "] at term %" PRId64 "", pRaft->selfGroupId, pRaft->selfId, lastTerm, lastIndex, pRaft->voteFor, grant ? "grant" : "reject", diff --git a/source/libs/sync/src/sync.c b/source/libs/sync/src/sync.c index e49b1d7983..321b03d2ee 100644 --- a/source/libs/sync/src/sync.c +++ b/source/libs/sync/src/sync.c @@ -23,8 +23,8 @@ SSyncManager* gSyncManager = NULL; #define SYNC_ACTIVITY_TIMER 5 #define SYNC_SERVER_WORKER 2 -static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet); -static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyncCfg); static int syncInitRpcClient(SSyncManager* syncManager); @@ -119,7 +119,7 @@ SSyncNode* syncStart(const SSyncInfo* pInfo) { return NULL; } - pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pInfo->vgId, gSyncManager->syncTimerManager); + pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)((int64_t)pInfo->vgId), gSyncManager->syncTimerManager); // start raft pNode->raft.pNode = pNode; @@ -176,12 +176,12 @@ int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode) { } // process rpc rsp message from other sync server -static void syncProcessRsp(SRpcMsg *pMsg, SEpSet *pEpSet) { +static void syncProcessRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } // process rpc message from other sync server -static void syncProcessReqMsg(SRpcMsg *pMsg, SEpSet *pEpSet) { +static void syncProcessReqMsg(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } @@ -195,7 +195,7 @@ static int syncInitRpcServer(SSyncManager* syncManager, const SSyncCluster* pSyn } assert(pSyncCfg->selfIndex < pSyncCfg->replica && pSyncCfg->selfIndex >= 0); const SNodeInfo* pNode = &(pSyncCfg->nodeInfo[pSyncCfg->replica]); - char buffer[20] = {'\0'}; + char buffer[156] = {'\0'}; snprintf(buffer, sizeof(buffer), "%s:%d", &(pNode->nodeFqdn[0]), pNode->nodePort); size_t len = strlen(buffer); void** ppRpcServer = taosHashGet(gSyncManager->rpcServerTable, buffer, len); @@ -287,7 +287,7 @@ static void *syncWorkerMain(void *argv) { } static void syncNodeTick(void *param, void *tmrId) { - SyncGroupId vgId = (SyncGroupId)param; + SyncGroupId vgId = (SyncGroupId)((int64_t)param); SSyncNode **ppNode = taosHashGet(gSyncManager->vgroupTable, &vgId, sizeof(SyncGroupId*)); if (ppNode == NULL) { return; @@ -298,5 +298,5 @@ static void syncNodeTick(void *param, void *tmrId) { syncRaftTick(&pNode->raft); pthread_mutex_unlock(&pNode->mutex); - pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)pNode->vgId, gSyncManager->syncTimerManager); + pNode->syncTimer = taosTmrStart(syncNodeTick, SYNC_TICK_TIMER, (void*)(int64_t)pNode->vgId, gSyncManager->syncTimerManager); } \ No newline at end of file diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 623293f82b..9002879b10 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -118,7 +118,11 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { pDiskId->id = -1; if (pDiskId->level >= pTfs->nlevel) { - pDiskId->level--; + pDiskId->level = pTfs->nlevel - 1; + } + + if (pDiskId->level < 0) { + pDiskId->level = 0; } while (pDiskId->level >= 0) { @@ -289,7 +293,7 @@ int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { STfsDisk *pDisk = pTier->disks[id]; snprintf(oaname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, orname); snprintf(naname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, nrname); - if (taosRenameFile(oaname, naname) != 0) { + if (taosRenameFile(oaname, naname) != 0 && errno != ENOENT) { terrno = TAOS_SYSTEM_ERROR(errno); fError("failed to rename %s to %s since %s", oaname, naname, terrstr()); return -1; diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index 270fff9ff3..e4390d13d1 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -69,7 +69,7 @@ STfsDisk *tfsMountDiskToTier(STfsTier *pTier, SDiskCfg *pCfg) { pTier->disks[id] = pDisk; pTier->ndisk++; - fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); + fDebug("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); return pTier->disks[id]; } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index abc00c5cd9..178d115c59 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -246,29 +246,13 @@ TEST_F(TfsTest, 04_File) { snprintf(fulldir, 128, "%s%s%s", root, TD_DIRSEP, "t3"); EXPECT_STREQ(dir, fulldir); - EXPECT_NE(tfsCopyFile(&f1, &f2), 0); + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); char af2[128] = {0}; snprintf(af2, 128, "%s%s%s", root, TD_DIRSEP, n2); EXPECT_EQ(taosDirExist(af2), 0); tfsRemoveFile(&f2); EXPECT_NE(taosDirExist(af2), 0); - EXPECT_NE(tfsCopyFile(&f1, &f2), 0); - - { - STfsDir *pDir = tfsOpendir(pTfs, ""); - - const STfsFile *pf1 = tfsReaddir(pDir); - EXPECT_STREQ(pf1->rname, "t3"); - EXPECT_EQ(pf1->did.id, 0); - EXPECT_EQ(pf1->did.level, 0); - EXPECT_EQ(pf1->pTfs, pTfs); - - const STfsFile *pf2 = tfsReaddir(pDir); - EXPECT_EQ(pf2, nullptr); - - tfsClosedir(pDir); - } { STfsDir *pDir = tfsOpendir(pTfs, "t3"); @@ -280,7 +264,26 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(pf1->pTfs, pTfs); const STfsFile *pf2 = tfsReaddir(pDir); - EXPECT_NE(pf2, nullptr); + EXPECT_EQ(pf2, nullptr); + + tfsClosedir(pDir); + } + + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_EQ(pf1->did.id, 0); + EXPECT_EQ(pf1->did.level, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_EQ(pf2->did.id, 0); + EXPECT_EQ(pf2->did.level, 0); + EXPECT_EQ(pf2->pTfs, pTfs); const STfsFile *pf3 = tfsReaddir(pDir); EXPECT_EQ(pf3, nullptr); @@ -289,5 +292,415 @@ TEST_F(TfsTest, 04_File) { } } + tfsClose(pTfs); +} + +TEST_F(TfsTest, 05_MultiDisk) { + int32_t code = 0; + + const char *root00 = "/tmp/tfsTest00"; + const char *root01 = "/tmp/tfsTest01"; + const char *root10 = "/tmp/tfsTest10"; + const char *root11 = "/tmp/tfsTest11"; + const char *root12 = "/tmp/tfsTest12"; + const char *root20 = "/tmp/tfsTest20"; + const char *root21 = "/tmp/tfsTest21"; + const char *root22 = "/tmp/tfsTest22"; + const char *root23 = "/tmp/tfsTest23"; + + SDiskCfg dCfg[9] = {0}; + tstrncpy(dCfg[0].dir, root01, TSDB_FILENAME_LEN); + dCfg[0].level = 0; + dCfg[0].primary = 0; + tstrncpy(dCfg[1].dir, root00, TSDB_FILENAME_LEN); + dCfg[1].level = 0; + dCfg[1].primary = 0; + tstrncpy(dCfg[2].dir, root20, TSDB_FILENAME_LEN); + dCfg[2].level = 2; + dCfg[2].primary = 0; + tstrncpy(dCfg[3].dir, root21, TSDB_FILENAME_LEN); + dCfg[3].level = 2; + dCfg[3].primary = 0; + tstrncpy(dCfg[4].dir, root22, TSDB_FILENAME_LEN); + dCfg[4].level = 2; + dCfg[4].primary = 0; + tstrncpy(dCfg[5].dir, root23, TSDB_FILENAME_LEN); + dCfg[5].level = 2; + dCfg[5].primary = 0; + tstrncpy(dCfg[6].dir, root10, TSDB_FILENAME_LEN); + dCfg[6].level = 1; + dCfg[6].primary = 0; + tstrncpy(dCfg[7].dir, root11, TSDB_FILENAME_LEN); + dCfg[7].level = 1; + dCfg[7].primary = 0; + tstrncpy(dCfg[8].dir, root12, TSDB_FILENAME_LEN); + dCfg[8].level = 1; + dCfg[8].primary = 0; + + taosRemoveDir(root00); + taosRemoveDir(root01); + taosRemoveDir(root10); + taosRemoveDir(root11); + taosRemoveDir(root12); + taosRemoveDir(root20); + taosRemoveDir(root21); + taosRemoveDir(root22); + taosRemoveDir(root23); + taosMkDir(root00); + taosMkDir(root01); + taosMkDir(root10); + taosMkDir(root11); + taosMkDir(root12); + taosMkDir(root20); + taosMkDir(root21); + taosMkDir(root22); + taosMkDir(root23); + + STfs *pTfs = tfsOpen(dCfg, 9); + ASSERT_EQ(pTfs, nullptr); + + dCfg[0].primary = 1; + dCfg[1].primary = 1; + pTfs = tfsOpen(dCfg, 9); + ASSERT_EQ(pTfs, nullptr); + + dCfg[0].primary = 0; + dCfg[1].primary = 1; + pTfs = tfsOpen(dCfg, 9); + ASSERT_NE(pTfs, nullptr); + + tfsUpdateSize(pTfs); + SDiskSize size = tfsGetSize(pTfs); + + EXPECT_GT(size.avail, 0); + EXPECT_GT(size.used, 0); + EXPECT_GT(size.total, size.avail); + EXPECT_GT(size.total, size.used); + + //------------- AllocDisk -----------------// + { + const char *path = NULL; + SDiskID did; + did.id = 0; + did.level = 0; + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root00); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root01); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root00); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root01); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root00); + + code = tfsAllocDisk(pTfs, 0, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 0); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root01); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root10); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root11); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 2); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root12); + + code = tfsAllocDisk(pTfs, 1, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 1); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root10); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root20); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root21); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 2); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root22); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 3); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root23); + + code = tfsAllocDisk(pTfs, 2, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 0); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root20); + + code = tfsAllocDisk(pTfs, 3, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 1); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root21); + + code = tfsAllocDisk(pTfs, 4, &did); + EXPECT_EQ(code, 0); + EXPECT_EQ(did.id, 2); + EXPECT_EQ(did.level, 2); + path = tfsGetDiskPath(pTfs, did); + EXPECT_STREQ(path, root22); + + const char *primary = tfsGetPrimaryPath(pTfs); + EXPECT_STREQ(primary, root00); + } + + //------------- Dir -----------------// + { + char p1[] = "p1"; + char ap00[128] = {0}; + snprintf(ap00, 128, "%s%s%s", root00, TD_DIRSEP, p1); + char ap01[128] = {0}; + snprintf(ap01, 128, "%s%s%s", root01, TD_DIRSEP, p1); + char ap10[128] = {0}; + snprintf(ap10, 128, "%s%s%s", root10, TD_DIRSEP, p1); + char ap11[128] = {0}; + snprintf(ap11, 128, "%s%s%s", root11, TD_DIRSEP, p1); + char ap12[128] = {0}; + snprintf(ap12, 128, "%s%s%s", root12, TD_DIRSEP, p1); + char ap20[128] = {0}; + snprintf(ap20, 128, "%s%s%s", root20, TD_DIRSEP, p1); + char ap21[128] = {0}; + snprintf(ap21, 128, "%s%s%s", root21, TD_DIRSEP, p1); + char ap22[128] = {0}; + snprintf(ap22, 128, "%s%s%s", root22, TD_DIRSEP, p1); + char ap23[128] = {0}; + snprintf(ap23, 128, "%s%s%s", root23, TD_DIRSEP, p1); + EXPECT_NE(taosDirExist(ap00), 0); + EXPECT_NE(taosDirExist(ap01), 0); + EXPECT_NE(taosDirExist(ap10), 0); + EXPECT_NE(taosDirExist(ap11), 0); + EXPECT_NE(taosDirExist(ap12), 0); + EXPECT_NE(taosDirExist(ap20), 0); + EXPECT_NE(taosDirExist(ap21), 0); + EXPECT_NE(taosDirExist(ap22), 0); + EXPECT_NE(taosDirExist(ap23), 0); + EXPECT_EQ(tfsMkdir(pTfs, p1), 0); + EXPECT_EQ(taosDirExist(ap00), 0); + EXPECT_EQ(taosDirExist(ap01), 0); + EXPECT_EQ(taosDirExist(ap10), 0); + EXPECT_EQ(taosDirExist(ap11), 0); + EXPECT_EQ(taosDirExist(ap12), 0); + EXPECT_EQ(taosDirExist(ap20), 0); + EXPECT_EQ(taosDirExist(ap21), 0); + EXPECT_EQ(taosDirExist(ap22), 0); + EXPECT_EQ(taosDirExist(ap23), 0); + EXPECT_EQ(tfsRmdir(pTfs, p1), 0); + EXPECT_NE(taosDirExist(ap00), 0); + EXPECT_NE(taosDirExist(ap01), 0); + EXPECT_NE(taosDirExist(ap10), 0); + EXPECT_NE(taosDirExist(ap11), 0); + EXPECT_NE(taosDirExist(ap12), 0); + EXPECT_NE(taosDirExist(ap20), 0); + EXPECT_NE(taosDirExist(ap21), 0); + EXPECT_NE(taosDirExist(ap22), 0); + EXPECT_NE(taosDirExist(ap23), 0); + + char p2[] = "p2"; + char _ap21[128] = {0}; + snprintf(_ap21, 128, "%s%s%s", root21, TD_DIRSEP, p2); + SDiskID did = {0}; + did.level = 2; + did.id = 1; + EXPECT_NE(taosDirExist(_ap21), 0); + EXPECT_EQ(tfsMkdirAt(pTfs, p2, did), 0); + EXPECT_EQ(taosDirExist(_ap21), 0); + + char p3[] = "p3/p2/p1/p0"; + char _ap12[128] = {0}; + snprintf(_ap12, 128, "%s%s%s", root12, TD_DIRSEP, p3); + did.level = 1; + did.id = 2; + EXPECT_NE(taosDirExist(_ap12), 0); + EXPECT_NE(tfsMkdir(pTfs, p3), 0); + EXPECT_NE(tfsMkdirAt(pTfs, p3, did), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p3, did), 0); + EXPECT_EQ(taosDirExist(_ap12), 0); + EXPECT_EQ(tfsRmdir(pTfs, p3), 0); + EXPECT_NE(taosDirExist(_ap12), 0); + + char p45[] = "p5"; + char p44[] = "p4"; + char p4[] = "p4/p2/p1/p0"; + char _ap22[128] = {0}; + snprintf(_ap22, 128, "%s%s%s", root22, TD_DIRSEP, p4); + did.level = 2; + did.id = 2; + + EXPECT_NE(taosDirExist(_ap22), 0); + EXPECT_EQ(tfsMkdirRecurAt(pTfs, p4, did), 0); + EXPECT_EQ(taosDirExist(_ap22), 0); + EXPECT_EQ(tfsRename(pTfs, p44, p45), 0); + EXPECT_EQ(tfsRmdir(pTfs, p4), 0); + EXPECT_NE(taosDirExist(_ap22), 0); + } + + //------------- File -----------------// + { + STfsFile file0; + STfsFile file1; + STfsFile file2; + STfsFile file3; + STfsFile file4; + SDiskID did0 = {0}; + SDiskID did1 = {0}; + SDiskID did2 = {0}; + SDiskID did3 = {0}; + SDiskID did4 = {0}; + did3.id = 1; + did4.level = 1; + tfsInitFile(pTfs, &file0, did0, "fname"); + tfsInitFile(pTfs, &file1, did1, "fname"); + tfsInitFile(pTfs, &file2, did2, "fnamex"); + tfsInitFile(pTfs, &file3, did3, "fname"); + tfsInitFile(pTfs, &file4, did4, "fname"); + + EXPECT_TRUE(tfsIsSameFile(&file0, &file1)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file2)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file3)); + EXPECT_FALSE(tfsIsSameFile(&file0, &file4)); + + { + char n1[] = "t3/t1.json"; + char n2[] = "t3/t2.json"; + STfsFile f1 = {0}; + STfsFile f2 = {0}; + SDiskID did; + did1.level = 1; + did1.id = 2; + did2.level = 2; + did2.id = 3; + + tfsInitFile(pTfs, &f1, did1, n1); + tfsInitFile(pTfs, &f2, did2, n2); + + EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); + + FILE *fp = fopen(f1.aname, "w"); + ASSERT_NE(fp, nullptr); + fwrite("12345678", 1, 5, fp); + fclose(fp); + + char base[128] = {0}; + tfsBasename(&f1, base); + char dir[128] = {0}; + tfsDirname(&f1, dir); + + EXPECT_STREQ(base, "t1.json"); + + char fulldir[128]; + snprintf(fulldir, 128, "%s%s%s", root12, TD_DIRSEP, "t3"); + EXPECT_STREQ(dir, fulldir); + + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); + + char af2[128] = {0}; + snprintf(af2, 128, "%s%s%s", root23, TD_DIRSEP, n2); + EXPECT_EQ(taosDirExist(af2), 0); + tfsRemoveFile(&f2); + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_EQ(pf1->did.level, 1); + EXPECT_EQ(pf1->did.id, 2); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_EQ(pf2, nullptr); + + tfsClosedir(pDir); + } + + EXPECT_NE(taosDirExist(af2), 0); + EXPECT_GT(tfsCopyFile(&f1, &f2), 0); + + { + STfsDir *pDir = tfsOpendir(pTfs, "t3"); + + const STfsFile *pf1 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_GT(pf1->did.level, 0); + EXPECT_GT(pf1->did.id, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf2 = tfsReaddir(pDir); + EXPECT_NE(pf1, nullptr); + EXPECT_GT(pf1->did.level, 0); + EXPECT_GT(pf1->did.id, 0); + EXPECT_EQ(pf1->pTfs, pTfs); + + const STfsFile *pf3 = tfsReaddir(pDir); + EXPECT_EQ(pf3, nullptr); + + tfsClosedir(pDir); + } + } + } + tfsClose(pTfs); } \ No newline at end of file diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index f381768a34..d870ae98ab 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -752,8 +752,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, - hashstr); + tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, sid, + hashstr, pConn->spi); } return pConn; @@ -1612,7 +1612,7 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { } } } else { - tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); + tError("%s, auth spi:%d not matched with received:%d %p", pConn->info, pConn->spi, pHead->spi, pConn); code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 4b490936cc..bc9a9de318 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -32,6 +32,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->cfp = pInit->cfp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->connType = pInit->connType; + pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); return pRpc; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f2d844f73d..bfadfe56ef 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,10 +17,13 @@ #include "transComm.h" +#define CONN_PERSIST_TIME(para) (para * 1000 * 10) + typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; uv_write_t* writeReq; + void* hostThrd; SConnBuffer readBuf; void* data; queue conn; @@ -41,11 +44,11 @@ typedef struct SCliThrdObj { uv_loop_t* loop; uv_async_t* cliAsync; // uv_timer_t* pTimer; - void* cache; // conn pool + void* pool; // conn pool queue msg; pthread_mutex_t msgMtx; uint64_t nextTimeout; // next timeout - void* shandle; // + void* pTransInst; // } SCliThrdObj; @@ -62,15 +65,15 @@ typedef struct SConnList { // conn pool // add expire timeout and capacity limit -static void* connCacheCreate(int size); -static void* connCacheDestroy(void* cache); -static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); -static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); +static void* creatConnPool(int size); +static void* destroyConnPool(void* pool); +static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port); +static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn); // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// process data read from server, auth/decompress etc -static void clientProcessData(SCliConn* conn); +// process data read from server, auth/decompress etc later +static void clientHandleResp(SCliConn* conn); // check whether already read complete packet from server static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read @@ -83,78 +86,94 @@ static void clientWriteCb(uv_write_t* req, int status); static void clientConnCb(uv_connect_t* req, int status); static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); -static void clientConnDestroy(SCliConn* pConn); +static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void clientMsgDestroy(SCliMsg* pMsg); +// handle req from app +static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +// thread obj +static SCliThrdObj* createThrdObj(); +static void destroyThrdObj(SCliThrdObj* pThrd); +// thread static void* clientThread(void* arg); -static void clientProcessData(SCliConn* conn) { +static void clientHandleResp(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = pCtx->ahandle; + SRpcInfo* pRpc = pCtx->pRpc; SRpcMsg rpcMsg; rpcMsg.pCont = conn->readBuf.buf; rpcMsg.contLen = conn->readBuf.len; rpcMsg.ahandle = pCtx->ahandle; (pRpc->cfp)(NULL, &rpcMsg, NULL); + + SCliThrdObj* pThrd = conn->hostThrd; + addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { + uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); + } + free(pCtx->ip); + free(pCtx); // impl } -static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); static void clientTimeoutCb(uv_timer_t* handle) { SCliThrdObj* pThrd = handle->data; - SRpcInfo* pRpc = pThrd->shandle; + SRpcInfo* pRpc = pThrd->pTransInst; int64_t currentTime = pThrd->nextTimeout; + tDebug("timeout, try to remove expire conn from conn pool"); - SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); + SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); while (p != NULL) { while (!QUEUE_IS_EMPTY(&p->conn)) { queue* h = QUEUE_HEAD(&p->conn); SCliConn* c = QUEUE_DATA(h, SCliConn, conn); if (c->expireTime < currentTime) { QUEUE_REMOVE(h); - clientConnDestroy(c); + // uv_stream_t stm = *(c->stream); + // uv_close((uv_handle_t*)&stm, clientDestroy); + clientConnDestroy(c, true); } else { break; } } - p = taosHashIterate((SHashObj*)pThrd->cache, p); + p = taosHashIterate((SHashObj*)pThrd->pool, p); } - pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; - uv_timer_start(handle, clientTimeoutCb, pRpc->idleTime * 10, 0); + pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); + uv_timer_start(handle, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } -static void* connCacheCreate(int size) { - SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - return false; +static void* creatConnPool(int size) { + // thread local, no lock + return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); } -static void* connCacheDestroy(void* cache) { - SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); +static void* destroyConnPool(void* pool) { + SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { queue* h = QUEUE_HEAD(&connList->conn); QUEUE_REMOVE(h); SCliConn* c = QUEUE_DATA(h, SCliConn, conn); - clientConnDestroy(c); + clientConnDestroy(c, true); } - connList = taosHashIterate((SHashObj*)cache, connList); + connList = taosHashIterate((SHashObj*)pool, connList); } - taosHashClear(cache); + taosHashClear(pool); } -static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { +static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { char key[128] = {0}; tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); - SHashObj* pCache = cache; - SConnList* plist = taosHashGet(pCache, key, strlen(key)); + SHashObj* pPool = pool; + SConnList* plist = taosHashGet(pPool, key, strlen(key)); if (plist == NULL) { SConnList list; - plist = &list; + taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list)); + plist = taosHashGet(pPool, key, strlen(key)); QUEUE_INIT(&plist->conn); - taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist)); } if (QUEUE_IS_EMPTY(&plist->conn)) { @@ -164,15 +183,14 @@ static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { QUEUE_REMOVE(h); return QUEUE_DATA(h, SCliConn, conn); } -static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { +static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { char key[128] = {0}; tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); - STransConnCtx* ctx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = ctx->pRpc; - conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; - SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); + SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); + SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -200,10 +218,11 @@ static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); + pBuf->buf = (char*)calloc(1, CAPACITY * sizeof(char)); pBuf->len = 0; pBuf->cap = CAPACITY; pBuf->left = -1; + buf->base = pBuf->buf; buf->len = CAPACITY; } else { @@ -213,7 +232,7 @@ static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, pBuf->buf = realloc(pBuf->buf, pBuf->cap); } else if (pBuf->len + pBuf->left > pBuf->cap) { pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); + pBuf->buf = realloc(pBuf->buf, pBuf->cap); } } buf->base = pBuf->buf + pBuf->len; @@ -227,29 +246,38 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { - tDebug("alread read complete pack"); - clientProcessData(conn); + tDebug("alread read complete"); + clientHandleResp(conn); } else { - tDebug("read halp packet, continue to read"); + tDebug("read half packet, continue to read"); } return; } - - if (nread != UV_EOF) { - tDebug("Read error %s\n", uv_err_name(nread)); + assert(nread <= 0); + if (nread == 0) { + return; } - // - uv_close((uv_handle_t*)handle, clientDestroy); + if (nread != UV_EOF) { + tDebug("read error %s", uv_err_name(nread)); + } + // tDebug("Read error %s\n", uv_err_name(nread)); + // uv_close((uv_handle_t*)handle, clientDestroy); } -static void clientConnDestroy(SCliConn* conn) { - // impl later - // +static void clientConnDestroy(SCliConn* conn, bool clear) { + tDebug("conn %p destroy", conn); + if (clear) { + uv_close((uv_handle_t*)conn->stream, NULL); + } + free(conn->stream); + free(conn->readBuf.buf); + free(conn->writeReq); + free(conn); } static void clientDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - QUEUE_REMOVE(&conn->conn); - clientConnDestroy(conn); + // QUEUE_REMOVE(&conn->conn); + clientConnDestroy(conn, false); } static void clientWriteCb(uv_write_t* req, int status) { @@ -257,10 +285,16 @@ static void clientWriteCb(uv_write_t* req, int status) { if (status == 0) { tDebug("data already was written on stream"); } else { - uv_close((uv_handle_t*)pConn->stream, clientDestroy); + tError("failed to write: %s", uv_err_name(status)); + clientConnDestroy(pConn, true); return; } - + SCliThrdObj* pThrd = pConn->hostThrd; + if (pConn->stream == NULL) { + pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); + pConn->stream->data = pConn; + } uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } @@ -270,35 +304,37 @@ static void clientWrite(SCliConn* pConn) { SRpcMsg* pMsg = (SRpcMsg*)(&pCliMsg->msg); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - int msgLen = transMsgLenFromCont(pMsg->contLen); - char* msg = (char*)(pHead); + int msgLen = transMsgLenFromCont(pMsg->contLen); - uv_buf_t wb = uv_buf_init(msg, msgLen); + pHead->msgType = pMsg->msgType; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + + uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); + tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; - if (status != 0) { - tError("failed to connect %s", uv_err_name(status)); - clientConnDestroy(pConn); - return; - } + SCliMsg* pMsg = pConn->data; - SCliMsg* pMsg = pConn->data; - STransConnCtx* pCtx = ((SCliMsg*)(pConn->data))->ctx; - - SRpcMsg rpcMsg; - rpcMsg.ahandle = pCtx->ahandle; + STransConnCtx* pCtx = pMsg->ctx; + SRpcInfo* pRpc = pCtx->pRpc; if (status != 0) { + // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); + tError("failed to connect server, errmsg: %s", uv_strerror(status)); // call user fp later - tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); - SRpcInfo* pRpc = pMsg->ctx->pRpc; + SRpcMsg rpcMsg; + rpcMsg.ahandle = pCtx->ahandle; + // SRpcInfo* pRpc = pMsg->ctx->pRpc; (pRpc->cfp)(NULL, &rpcMsg, NULL); - uv_close((uv_handle_t*)req->handle, clientDestroy); + + clientConnDestroy(pConn, true); + // uv_close((uv_handle_t*)req->handle, clientDestroy); return; } + assert(pConn->stream == req->handle); clientWrite(pConn); } @@ -310,22 +346,32 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { et = taosGetTimestampUs(); STransConnCtx* pCtx = pMsg->ctx; - SCliConn* conn = getConnFromCache(pThrd->cache, pCtx->ip, pCtx->port); + SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later conn->data = pMsg; conn->writeReq->data = conn; + + conn->readBuf.len = 0; + memset(conn->readBuf.buf, 0, conn->readBuf.cap); + conn->readBuf.left = -1; clientWrite(conn); } else { SCliConn* conn = calloc(1, sizeof(SCliConn)); + // read/write stream handle conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); + conn->stream->data = conn; + + // write req handle conn->writeReq = malloc(sizeof(uv_write_t)); + conn->writeReq->data = conn; QUEUE_INIT(&conn->conn); conn->connReq.data = conn; conn->data = pMsg; + conn->hostThrd = pThrd; struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); @@ -351,42 +397,29 @@ static void clientAsyncCb(uv_async_t* handle) { SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); clientHandleReq(pMsg, pThrd); count++; - if (count >= 2) { - tError("send batch size: %d", count); - } + } + if (count >= 2) { + tDebug("already process batch size: %d", count); } } static void* clientThread(void* arg) { SCliThrdObj* pThrd = (SCliThrdObj*)arg; - SRpcInfo* pRpc = pThrd->shandle; - pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; - uv_timer_start(pThrd->pTimer, clientTimeoutCb, pRpc->idleTime * 10, 0); uv_run(pThrd->loop, UV_RUN_DEFAULT); } void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { SClientObj* cli = calloc(1, sizeof(SClientObj)); + SRpcInfo* pRpc = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); - QUEUE_INIT(&pThrd->msg); - pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pThrd->loop); - - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; - - pThrd->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pThrd->loop, pThrd->pTimer); - - pThrd->shandle = shandle; + SCliThrdObj* pThrd = createThrdObj(); + pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); + pThrd->pTransInst = shandle; int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); if (err == 0) { @@ -400,16 +433,41 @@ static void clientMsgDestroy(SCliMsg* pMsg) { // impl later free(pMsg); } +static SCliThrdObj* createThrdObj() { + SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); + QUEUE_INIT(&pThrd->msg); + pthread_mutex_init(&pThrd->msgMtx, NULL); + + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); + + pThrd->cliAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); + pThrd->cliAsync->data = pThrd; + + pThrd->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pThrd->pTimer); + pThrd->pTimer->data = pThrd; + + pThrd->pool = creatConnPool(1); + return pThrd; +} +static void destroyThrdObj(SCliThrdObj* pThrd) { + if (pThrd == NULL) { + return; + } + pthread_join(pThrd->thread, NULL); + pthread_mutex_destroy(&pThrd->msgMtx); + free(pThrd->cliAsync); + free(pThrd->loop); + free(pThrd); +} +// void taosCloseClient(void* arg) { // impl later SClientObj* cli = arg; for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = cli->pThreadObj[i]; - pthread_join(pThrd->thread, NULL); - pthread_mutex_destroy(&pThrd->msgMtx); - free(pThrd->cliAsync); - free(pThrd->loop); - free(pThrd); + destroyThrdObj(cli->pThreadObj[i]); } free(cli->pThreadObj); free(cli); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 4542541043..b519a35f24 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -14,8 +14,8 @@ */ #ifdef USE_UV -#include "transComm.h" +#include "transComm.h" typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -24,13 +24,15 @@ typedef struct SConn { uv_async_t* pWorkerAsync; queue queue; int ref; - int persist; // persist connection or not - SConnBuffer connBuf; // read buf, - SConnBuffer writeBuf; // write buf + int persist; // persist connection or not + SConnBuffer connBuf; // read buf, int count; - void* shandle; // rpc init - void* ahandle; // + int inType; + void* pTransInst; // rpc init + void* ahandle; // void* hostThrd; + + SRpcMsg sendMsg; // del later char secured; int spi; @@ -48,7 +50,7 @@ typedef struct SWorkThrdObj { uv_async_t* workerAsync; // queue conn; pthread_mutex_t connMtx; - void* shandle; + void* pTransInst; } SWorkThrdObj; typedef struct SServerObj { @@ -66,7 +68,7 @@ typedef struct SServerObj { static const char* notify = "a"; // refactor later -static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen); +static int transAddAuthPart(SConn* pConn, char* msg, int msgLen); static int uvAuthMsg(SConn* pConn, char* msg, int msgLen); @@ -75,21 +77,28 @@ static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_b static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); static void uvOnTimeoutCb(uv_timer_t* handle); static void uvOnWriteCb(uv_write_t* req, int status); +static void uvOnPipeWriteCb(uv_write_t* req, int status); static void uvOnAcceptCb(uv_stream_t* stream, int status); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvWorkerAsyncCb(uv_async_t* handle); -// already read complete packet -static bool readComplete(SConnBuffer* buf); +static void uvPrepareSendData(SConn* conn, uv_buf_t* wb); -static SConn* connCreate(); -static void connDestroy(SConn* conn); -static void uvConnDestroy(uv_handle_t* handle); +// check whether already read complete packet +static bool readComplete(SConnBuffer* buf); +static SConn* createConn(); +static void destroyConn(SConn* conn, bool clear /*clear handle or not*/); -// server worke thread +static void uvDestroyConn(uv_handle_t* handle); + +// server and worker thread static void* workerThread(void* arg); static void* acceptThread(void* arg); +// add handle loop +static bool addHandleToWorkloop(void* arg); +static bool addHandleToAcceptloop(void* arg); + void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { /* * formate of data buffer: @@ -135,25 +144,28 @@ static bool readComplete(SConnBuffer* data) { if (msgLen > data->len) { data->left = msgLen - data->len; return false; - } else { + } else if (msgLen == data->len) { return true; + } else if (msgLen < data->len) { + return false; + // handle other packet later } } else { return false; } } -static void uvDoProcess(SRecvInfo* pRecv) { - // impl later - STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; - SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; - SConn* pConn = pRecv->thandle; - tDump(pRecv->msg, pRecv->msgLen); - terrno = 0; - // SRpcReqContext* pContest; - - // do auth and check -} +// static void uvDoProcess(SRecvInfo* pRecv) { +// // impl later +// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; +// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; +// SConn* pConn = pRecv->thandle; +// tDump(pRecv->msg, pRecv->msgLen); +// terrno = 0; +// // SRpcReqContext* pContest; +// +// // do auth and check +//} static int uvAuthMsg(SConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; @@ -222,12 +234,13 @@ static void uvProcessData(SConn* pConn) { p->msgLen = pBuf->len; p->ip = 0; p->port = 0; - p->shandle = pConn->shandle; // + p->shandle = pConn->pTransInst; // p->thandle = pConn; p->chandle = NULL; - // STransMsgHead* pHead = (STransMsgHead*)p->msg; + + pConn->inType = pHead->msgType; assert(transIsReq(pHead->msgType)); SRpcInfo* pRpc = (SRpcInfo*)p->shandle; @@ -247,7 +260,9 @@ static void uvProcessData(SConn* pConn) { // add compress later // pHead = rpcDecompressRpcMsg(pHead); } else { + pHead->msgLen = htonl(pHead->msgLen); // impl later + // } rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; @@ -257,7 +272,7 @@ static void uvProcessData(SConn* pConn) { rpcMsg.handle = pConn; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); - uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0); + // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth // validate msg type } @@ -268,6 +283,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; + tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread); if (readComplete(pBuf)) { tDebug("alread read complete packet"); uvProcessData(conn); @@ -276,10 +292,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } - if (nread != UV_EOF) { - tDebug("Read error %s\n", uv_err_name(nread)); + if (nread == 0) { + return; + } + if (nread != UV_EOF) { + tDebug("read error %s", uv_err_name(nread)); } - uv_close((uv_handle_t*)cli, uvConnDestroy); } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->base = malloc(sizeof(char)); @@ -293,16 +311,49 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnWriteCb(uv_write_t* req, int status) { SConn* conn = req->data; + + SConnBuffer* buf = &conn->connBuf; + buf->len = 0; + memset(buf->buf, 0, buf->cap); + buf->left = -1; if (status == 0) { tDebug("data already was written on stream"); } else { - connDestroy(conn); + tDebug("failed to write data, %s", uv_err_name(status)); + destroyConn(conn, true); } // opt } +static void uvOnPipeWriteCb(uv_write_t* req, int status) { + if (status == 0) { + tDebug("success to dispatch conn to work thread"); + } else { + tError("fail to dispatch conn to work thread"); + } +} +static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { + // impl later; + tDebug("prepare to send back"); + SRpcMsg* pMsg = &conn->sendMsg; + if (pMsg->pCont == 0) { + pMsg->pCont = (void*)rpcMallocCont(0); + pMsg->contLen = 0; + } + STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); + pHead->msgType = conn->inType + 1; + // add more info + char* msg = (char*)pHead; + int32_t len = transMsgLenFromCont(pMsg->contLen); + if (transCompressMsg(msg, len, NULL)) { + // impl later + } + pHead->msgLen = htonl(len); + wb->base = msg; + wb->len = len; +} void uvWorkerAsyncCb(uv_async_t* handle) { - SWorkThrdObj* pThrd = container_of(handle, SWorkThrdObj, workerAsync); + SWorkThrdObj* pThrd = handle->data; SConn* conn = NULL; queue wq; // batch process to avoid to lock/unlock frequently @@ -318,8 +369,8 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("except occurred, do nothing"); return; } - uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); - + uv_buf_t wb; + uvPrepareSendData(conn, &wb); uv_timer_stop(conn->pTimer); uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); @@ -341,10 +392,12 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThreads; + tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); - uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); + free(cli); } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { @@ -354,7 +407,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tError("read error %s", uv_err_name(nread)); } // TODO(log other failure reason) - uv_close((uv_handle_t*)q, NULL); + // uv_close((uv_handle_t*)q, NULL); return; } // free memory allocated by @@ -373,8 +426,8 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConn* pConn = connCreate(); - pConn->shandle = pThrd->shandle; + SConn* pConn = createConn(); + pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pConn->pTimer); @@ -398,34 +451,25 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tDebug("new connection created: %d", fd); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { - connDestroy(pConn); + tDebug("failed to create new connection"); + destroyConn(pConn, true); } } void* acceptThread(void* arg) { // opt SServerObj* srv = (SServerObj*)arg; - uv_tcp_init(srv->loop, &srv->server); - - struct sockaddr_in bind_addr; - - uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); - uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); - int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { - tError("Listen error %s\n", uv_err_name(err)); - return NULL; - } uv_run(srv->loop, UV_RUN_DEFAULT); } -void* workerThread(void* arg) { - SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; - +static bool addHandleToWorkloop(void* arg) { + SWorkThrdObj* pThrd = arg; pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pThrd->loop); + if (0 != uv_loop_init(pThrd->loop)) { + return false; + } // SRpcInfo* pRpc = pThrd->shandle; - uv_pipe_init(pThrd->loop, pThrd->pipe, 0); + uv_pipe_init(pThrd->loop, pThrd->pipe, 1); uv_pipe_open(pThrd->pipe, pThrd->fd); pThrd->pipe->data = pThrd; @@ -435,43 +479,79 @@ void* workerThread(void* arg) { pThrd->workerAsync = malloc(sizeof(uv_async_t)); uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); + pThrd->workerAsync->data = pThrd; uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + return true; +} + +static bool addHandleToAcceptloop(void* arg) { + // impl later + SServerObj* srv = arg; + + int err = 0; + if ((err = uv_tcp_init(srv->loop, &srv->server)) != 0) { + tError("failed to init accept server: %s", uv_err_name(err)); + return false; + } + + struct sockaddr_in bind_addr; + + uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); + if ((err = uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0)) != 0) { + tError("failed to bind: %s", uv_err_name(err)); + return false; + } + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + tError("failed to listen: %s", uv_err_name(err)); + return false; + } + return true; +} +void* workerThread(void* arg) { + SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); } -static SConn* connCreate() { +static SConn* createConn() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); return pConn; } -static void connDestroy(SConn* conn) { +static void connCloseCb(uv_handle_t* handle) { + // impl later + // +} +static void destroyConn(SConn* conn, bool clear) { if (conn == NULL) { return; } + if (clear) { + uv_handle_t handle = *((uv_handle_t*)conn->pTcp); + uv_close(&handle, NULL); + } uv_timer_stop(conn->pTimer); free(conn->pTimer); - uv_close((uv_handle_t*)conn->pTcp, NULL); - free(conn->connBuf.buf); free(conn->pTcp); + free(conn->connBuf.buf); free(conn->pWriter); free(conn); - // handle } -static void uvConnDestroy(uv_handle_t* handle) { +static void uvDestroyConn(uv_handle_t* handle) { SConn* conn = handle->data; - connDestroy(conn); + destroyConn(conn, false); } -static int rpcAddAuthPart(SConn* pConn, char* msg, int msgLen) { - SRpcHead* pHead = (SRpcHead*)msg; +static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) { + STransMsgHead* pHead = (STransMsgHead*)msg; if (pConn->spi && pConn->secured == 0) { // add auth part pHead->spi = pConn->spi; - SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen); + STransDigestMsg* pDigest = (STransDigestMsg*)(msg + msgLen); pDigest->timeStamp = htonl(taosGetTimestampSec()); msgLen += sizeof(SRpcDigest); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); - rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + // transBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); } else { pHead->spi = 0; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); @@ -493,18 +573,23 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); + srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); int fds[2]; if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { - return NULL; + goto End; } uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - thrd->shandle = shandle; + thrd->pTransInst = shandle; thrd->fd = fds[0]; thrd->pipe = &(srv->pipe[i][1]); // init read + + if (false == addHandleToWorkloop(thrd)) { + goto End; + } int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); if (err == 0) { tDebug("sucess to create worker-thread %d", i); @@ -513,9 +598,10 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, // TODO: clear all other resource later tError("failed to create worker-thread %d", i); } - srv->pThreadObj[i] = thrd; } - + if (false == addHandleToAcceptloop(srv)) { + goto End; + } int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); if (err == 0) { tDebug("success to create accept-thread"); @@ -524,16 +610,25 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } return srv; +End: + taosCloseServer(srv); + return NULL; +} + +void destroyWorkThrd(SWorkThrdObj* pThrd) { + if (pThrd == NULL) { + return; + } + pthread_join(pThrd->thread, NULL); + // free(srv->pipe[i]); + free(pThrd->loop); + free(pThrd); } void taosCloseServer(void* arg) { // impl later SServerObj* srv = arg; for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* pThrd = srv->pThreadObj[i]; - pthread_join(pThrd->thread, NULL); - free(srv->pipe[i]); - free(pThrd->loop); - free(pThrd); + destroyWorkThrd(srv->pThreadObj[i]); } free(srv->loop); free(srv->pipe); @@ -547,6 +642,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SWorkThrdObj* pThrd = pConn->hostThrd; // opt later + pConn->sendMsg = *pMsg; pthread_mutex_lock(&pThrd->connMtx); QUEUE_PUSH(&pThrd->conn, &pConn->queue); pthread_mutex_unlock(&pThrd->connMtx); diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 4ccbb60cc2..fd3496cc17 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -34,8 +34,8 @@ typedef struct { static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - // tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - // pMsg->code); + tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -63,6 +63,8 @@ static void *sendRequest(void *param) { if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); + tDebug("recv response"); + // usleep(100000000); } tDebug("thread:%d, it is over", pInfo->index); @@ -98,7 +100,7 @@ int main(int argc, char *argv[]) { rpcInit.numOfThreads = 1; rpcInit.cfp = processResponse; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.idleTime = 100; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 2e32aa57ca..12d8a01819 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -122,7 +122,7 @@ int main(int argc, char *argv[]) { rpcInit.numOfThreads = 1; rpcInit.cfp = processRequestMsg; rpcInit.sessions = 1000; - rpcInit.idleTime = tsShellActivityTimer * 1500; + rpcInit.idleTime = 2 * 1500; rpcInit.afp = retrieveAuthInfo; for (int i = 1; i < argc; ++i) { diff --git a/source/util/src/compare.c b/source/util/src/compare.c index f2d320fde0..a1c1625c34 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -16,13 +16,13 @@ #define _GNU_SOURCE #define _XOPEN_SOURCE #define _DEFAULT_SOURCE - #include "os.h" -#include "types.h" + #include "compare.h" -#include "ulog.h" -#include "thash.h" #include "regex.h" +#include "thash.h" +#include "types.h" +#include "ulog.h" int32_t setCompareBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -228,7 +228,7 @@ int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { * '_': Matches one character * */ -int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { +int32_t patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) { char c, c1; int32_t i = 0; @@ -289,7 +289,7 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { +int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, const SPatternCompareInfo *pInfo) { wchar_t c, c1; wchar_t matchOne = L'_'; // "_" wchar_t matchAll = L'%'; // "%" @@ -360,11 +360,11 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { memcpy(str, varDataVal(pLeft), sz); str[sz] = 0; - int errCode = 0; + int32_t errCode = 0; regex_t regex; char msgbuf[256] = {0}; - int cflags = REG_EXTENDED; + int32_t cflags = REG_EXTENDED; if ((errCode = regcomp(®ex, pattern, cflags)) != 0) { regerror(errCode, ®ex, msgbuf, sizeof(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf); diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 00263d7bce..328d3da5a4 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -13,11 +13,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tskiplist.h" + #include "os.h" + #include "compare.h" -#include "ulog.h" +#include "tskiplist.h" #include "tutil.h" +#include "ulog.h" static int initForwardBackwardPtr(SSkipList *pSkipList); static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); @@ -52,11 +54,17 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _ pSkipList->flags = flags; pSkipList->keyFn = fn; pSkipList->seed = rand(); + +#if 0 + // the function getkeycomparfunc is defined in common if (comparFn == NULL) { pSkipList->comparFn = getKeyComparFunc(keyType, TSDB_ORDER_ASC); } else { pSkipList->comparFn = comparFn; } +#else + pSkipList->comparFn = comparFn; +#endif if (initForwardBackwardPtr(pSkipList) < 0) { tSkipListDestroy(pSkipList); diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index b178ee0b10..1b13c102a5 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -4,6 +4,11 @@ #include "encode.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshift-count-overflow" +#pragma GCC diagnostic ignored "-Woverflow" +#pragma GCC diagnostic ignored "-Woverflow" + #define BUF_SIZE 64 td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN}; @@ -354,10 +359,10 @@ TEST(td_encode_test, compound_struct_encode_test) { SCoder encoder, decoder; uint8_t * buf1; int32_t buf1size; - uint8_t * buf2; + uint8_t *buf2; int32_t buf2size; - SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = "Hello"}; - SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = "Hello", .A_d = 67, .A_e = 13}; + SStructA_v1 sa1 = {.A_a = 10, .A_b = 65478, .A_c = (char *)"Hello"}; + SStructA_v2 sa2 = {.A_a = 10, .A_b = 65478, .A_c = (char *)"Hello", .A_d = 67, .A_e = 13}; SFinalReq_v1 req1 = {.pA = &sa1, .v_a = 15, .v_b = 35}; SFinalReq_v2 req2 = {.pA = &sa2, .v_a = 15, .v_b = 32, .v_c = 37}; SFinalReq_v1 dreq11, dreq21; @@ -430,4 +435,6 @@ TEST(td_encode_test, compound_struct_encode_test) { GTEST_ASSERT_EQ(dreq21.v_a, req2.v_a); GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b); tCoderClear(&decoder); -} \ No newline at end of file +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bb6569deb0..a27f5326c9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,17 +2,17 @@ #======================b1-start=============== # ---- user -./test.sh -f sim/user/basic1.sim +./test.sh -f tsim/user/basic1.sim # ---- db -./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/basic6.sim -./test.sh -f sim/db/error1.sim +./test.sh -f tsim/db/basic1.sim +./test.sh -f tsim/db/basic6.sim +./test.sh -f tsim/db/error1.sim # ---- table -./test.sh -f sim/table/basic1.sim +./test.sh -f tsim/table/basic1.sim # ---- dnode -./test.sh -f sim/dnode/basic1.sim +./test.sh -f tsim/dnode/basic1.sim #======================b1-end=============== diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index e00363b28f..ea1ad65a5b 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -113,8 +113,12 @@ elif [ $NODE -eq 7 ]; then NODE=7700 elif [ $NODE -eq 8 ]; then NODE=7800 +elif [ $NODE -eq 9 ]; then + NODE=7900 fi +HOSTNAME=localhost + echo " " >> $TAOS_CFG echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index e45c7724ba..37fdbaf784 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -70,7 +70,7 @@ createNewCfgFile() { echo "dataDir ${dataDir}" >> ${cfgFile} echo "logDir ${logDir}" >> ${cfgFile} echo "serverPort ${serverPort}" >> ${cfgFile} - + echo "numOfLogLines 100000000" >> ${cfgFile} echo "supportVnodes 1024" >> ${cfgFile} #echo "asyncLog 0" >> ${cfgFile} echo "telemetryReporting 0" >> ${cfgFile} diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index 28675be808..3b43656a41 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -5,7 +5,41 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 -system sh/cfg.sh -n dnode4 -c walLevel -v 2 +return + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/deploy.sh -n dnode6 -i 6 +system sh/deploy.sh -n dnode7 -i 7 +system sh/deploy.sh -n dnode8 -i 8 +system sh/deploy.sh -n dnode9 -i 9 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode6 -s start +system sh/exec.sh -n dnode7 -s start +system sh/exec.sh -n dnode8 -s start +system sh/exec.sh -n dnode9 -s start + +sleep 2000 + +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 +sql create dnode $hostname port 7600 +sql create dnode $hostname port 7700 +sql create dnode $hostname port 7800 +sql create dnode $hostname port 7900 + +sql show dnodes; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 +print $data40 $data41 \ No newline at end of file diff --git a/tests/script/sim/db/basic1.sim b/tests/script/tsim/db/basic1.sim similarity index 100% rename from tests/script/sim/db/basic1.sim rename to tests/script/tsim/db/basic1.sim diff --git a/tests/script/sim/db/basic6.sim b/tests/script/tsim/db/basic6.sim similarity index 100% rename from tests/script/sim/db/basic6.sim rename to tests/script/tsim/db/basic6.sim diff --git a/tests/script/sim/db/error1.sim b/tests/script/tsim/db/error1.sim similarity index 100% rename from tests/script/sim/db/error1.sim rename to tests/script/tsim/db/error1.sim diff --git a/tests/script/sim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim similarity index 100% rename from tests/script/sim/dnode/basic1.sim rename to tests/script/tsim/dnode/basic1.sim diff --git a/tests/script/sim/table/basic1.sim b/tests/script/tsim/table/basic1.sim similarity index 100% rename from tests/script/sim/table/basic1.sim rename to tests/script/tsim/table/basic1.sim diff --git a/tests/script/sim/user/basic1.sim b/tests/script/tsim/user/basic1.sim similarity index 100% rename from tests/script/sim/user/basic1.sim rename to tests/script/tsim/user/basic1.sim diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index b8d8123380..fddf3e7f2a 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -35,8 +35,7 @@ int32_t numOfVgroups = 2; int32_t showTablesFlag = 0; int32_t queryFlag = 0; -int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 - +int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 typedef struct { int64_t tableBeginIndex; @@ -89,7 +88,7 @@ void createDbAndStb() { pRes = taos_query(con, qstr); code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } taos_free_result(pRes); @@ -182,18 +181,14 @@ void *threadFunc(void *param) { exit(1); } - pError("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", - pInfo->threadIndex, - pInfo->tableBeginIndex, - pInfo->tableEndIndex); + pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + pInfo->tableEndIndex); pInfo->tableBeginIndex += startOffset; - pInfo->tableEndIndex += startOffset; + pInfo->tableEndIndex += startOffset; - pError("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", - pInfo->threadIndex, - pInfo->tableBeginIndex, - pInfo->tableEndIndex); + pPrint("====after thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); TAOS_RES *pRes = taos_query(con, qstr); @@ -221,7 +216,7 @@ void *threadFunc(void *param) { int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); code = taos_errno(pRes); - if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { + if (code != 0) { pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); } taos_free_result(pRes); @@ -251,20 +246,20 @@ void *threadFunc(void *param) { // batch = MIN(batch, batchNum); int32_t len = sprintf(qstr, "insert into "); - + for (int32_t i = 0; i < batchNumOfTbl;) { - int64_t ts = startTimestamp; + int64_t ts = startTimestamp; len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); for (int32_t j = 0; j < batchNumOfRow; j++) { - len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); } - + t++; i++; if (t > pInfo->tableEndIndex) { break; } - } + } int64_t startTs = taosGetTimestampUs(); TAOS_RES *pRes = taos_query(con, qstr); @@ -360,7 +355,7 @@ void parseArgument(int32_t argc, char *argv[]) { } else if (strcmp(argv[i], "-q") == 0) { queryFlag = atoi(argv[++i]); } else { - pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); } } @@ -390,15 +385,16 @@ int32_t main(int32_t argc, char *argv[]) { } if (queryFlag) { - //selectRowsFromTable(); + // selectRowsFromTable(); return 0; } if (createTable) { createDbAndStb(); } - - pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); + + pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, + startOffset); pthread_attr_t thattr; pthread_attr_init(&thattr); @@ -456,8 +452,8 @@ int32_t main(int32_t argc, char *argv[]) { if (createTable) { pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 - "us %s", - GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); + "us %s", + GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); } if (insertData) { diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 016b6500ed..6b6392c1be 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -42,7 +42,8 @@ char *simParseArbitratorName(char *varName) { char *simParseHostName(char *varName) { static char hostName[140]; - sprintf(hostName, "%s", simHostName); + //sprintf(hostName, "%s", simHostName); + sprintf(hostName, "%s", "localhost"); return hostName; } diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index 7b5dbd2405..be8714387f 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -319,7 +319,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) { char user[] = "nettestinternal"; char pass[] = "nettestinternal"; - taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); + taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt); memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; @@ -344,7 +344,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p SRpcMsg rspMsg; void * pRpcConn; - char secretEncrypt[32] = {0}; + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; pRpcConn = taosNetInitRpc(secretEncrypt, spi); if (NULL == pRpcConn) {