From d9daf767106073cbdc55f7afa2904632249e7192 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Oct 2022 10:18:28 +0800 Subject: [PATCH 01/60] fix: invalid read problem --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index efc74b68ba..3fee0951a0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = strlen(pVnode->path) + strlen(dir) + 2; + slen = TSDB_FILENAME_LEN; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -48,7 +48,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } pTsdb->path = (char *)&pTsdb[1]; - sprintf(pTsdb->path, "%s%s%s", pVnode->path, TD_DIRSEP, dir); + snprintf(pTsdb->path, TSDB_FILENAME_LEN, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); From cff83b883666c3769c702f4980f03c00776a99a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Oct 2022 17:17:05 +0800 Subject: [PATCH 02/60] disable userTest --- source/dnode/mnode/impl/test/user/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/test/user/CMakeLists.txt b/source/dnode/mnode/impl/test/user/CMakeLists.txt index b39ea0e73f..23c67bc6ca 100644 --- a/source/dnode/mnode/impl/test/user/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/user/CMakeLists.txt @@ -5,7 +5,7 @@ target_link_libraries( PUBLIC sut ) -add_test( - NAME userTest - COMMAND userTest -) +# add_test( +# NAME userTest +# COMMAND userTest +# ) From 8b7d2a4c9e2bc67e0ed1ea3378b600b5756eaa36 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 22 Oct 2022 09:03:29 +0800 Subject: [PATCH 03/60] test: comment out unstable case --- source/dnode/mnode/impl/test/db/CMakeLists.txt | 9 +++++---- source/libs/tfs/test/CMakeLists.txt | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt index 3f6a80835f..69e36cae08 100644 --- a/source/dnode/mnode/impl/test/db/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -5,7 +5,8 @@ target_link_libraries( PUBLIC sut ) -add_test( - NAME dbTest - COMMAND dbTest -) +# +#add_test( +# NAME dbTest +# COMMAND dbTest +#) diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt index ee28dcf723..977b982ebf 100644 --- a/source/libs/tfs/test/CMakeLists.txt +++ b/source/libs/tfs/test/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries( PUBLIC gtest_main ) -add_test( - NAME tfs_test - COMMAND tfs_test -) +#add_test( +# NAME tfs_test +# COMMAND tfs_test +#) From ed33fcb4069ea6c73640d0dcb10c776a73ec6ff7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 23 Oct 2022 08:38:03 +0800 Subject: [PATCH 04/60] fix: invalid read problem --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 3fee0951a0..bc7021d603 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = TSDB_FILENAME_LEN; + slen = PATH_MAX; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -48,7 +48,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } pTsdb->path = (char *)&pTsdb[1]; - snprintf(pTsdb->path, TSDB_FILENAME_LEN, "%s%s%s", pVnode->path, TD_DIRSEP, dir); + snprintf(pTsdb->path, PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); From 31a04218787ba7bfeadbc02c469bcc8bcd1cb288 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 27 Oct 2022 15:41:29 +0800 Subject: [PATCH 05/60] enh: check completeness of WAL log entries after repairing --- include/util/taoserror.h | 1 + source/libs/wal/src/walMeta.c | 32 ++++++++++++++++++++++++++++++++ source/util/src/terror.c | 1 + 3 files changed, 34 insertions(+) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index bbddb539c6..563820f60b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -451,6 +451,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) #define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) +#define TSDB_CODE_WAL_LOG_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x1007) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index bf3aef340c..671a59aef1 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -282,6 +282,33 @@ void walAlignVersions(SWal* pWal) { pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); } +bool walLogEntriesComplete(const SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + bool complete = true; + int32_t fileIdx = -1; + int64_t index = pWal->vers.firstVer; + + while (++fileIdx < sz) { + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + if (pFileInfo->firstVer != index) { + break; + } + index = pFileInfo->lastVer + ((fileIdx + 1 < sz) ? 1 : 0); + } + // empty is regarded as complete + if (sz != 0) { + complete = (index == pWal->vers.lastVer); + } + + if (!complete) { + wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], aligned with snaphotVer:%" PRId64, + pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer); + terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; + } + + return complete; +} + int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; @@ -395,6 +422,11 @@ int walCheckAndRepairMeta(SWal* pWal) { if (updateMeta) { (void)walSaveMeta(pWal); } + + if (!walLogEntriesComplete(pWal)) { + return -1; + } + return 0; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 37f9734eab..bbdf43c85e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -452,6 +452,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid versi TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") From 4fdc98d3fb4987fc87e67cb34fff943ca70589ef Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 28 Oct 2022 22:15:06 +0800 Subject: [PATCH 06/60] opt http module --- source/libs/transport/src/thttp.c | 92 ++++++++++++++++++++++++++-- source/libs/transport/src/transCli.c | 30 +-------- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 92989a45f5..6715290acf 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -20,11 +20,80 @@ #include "thttp.h" #include "taoserror.h" #include "tlog.h" +#include "transComm.h" // clang-format on #define HTTP_RECV_BUF_SIZE 1024 +typedef struct SHttpModule { + uv_loop_t* loop; + SAsyncPool* asyncPool; + TdThread thread; +} SHttpModule; + +typedef struct SHttpMsg { + queue q; + char* server; + int32_t port; + char* cont; + int32_t len; + EHttpCompFlag flag; + +} SHttpMsg; + +static TdThreadOnce transHttpInit = PTHREAD_ONCE_INIT; +static SHttpModule* http = NULL; + +static void* httpThread(void* arg) { + SHttpModule* http = (SHttpModule*)arg; + setThreadName("http-cli-send-thread"); + uv_run(http->loop, UV_RUN_DEFAULT); + return NULL; +} + +static void httpAsyncCb(uv_async_t* handle) { + SAsyncItem* item = handle->data; + SHttpModule* http = item->pThrd; + + SHttpMsg* msg = NULL; + + queue wq; + taosThreadMutexLock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + taosThreadMutexUnlock(&item->mtx); + + int count = 0; + while (!QUEUE_IS_EMPTY(&wq)) { + queue* h = QUEUE_HEAD(&wq); + QUEUE_REMOVE(h); + msg = QUEUE_DATA(h, SHttpMsg, q); + } +} + +static void transHttpEnvInit() { + http = taosMemoryMalloc(sizeof(SHttpModule)); + + http->loop = taosMemoryMalloc(sizeof(uv_loop_t)); + uv_loop_init(http->loop); + + http->asyncPool = transAsyncPoolCreate(http->loop, 1, http, httpAsyncCb); + + int err = taosThreadCreate(&http->thread, NULL, httpThread, (void*)http); + if (err != 0) { + taosMemoryFree(http->loop); + taosMemoryFree(http); + http = NULL; + } +} +static void transHttpEnvDestroy() { + if (http == NULL) return; + + transAsyncPoolDestroy(http->asyncPool); + taosMemoryFree(http->loop); + taosMemoryFree(http); +} + typedef struct SHttpClient { uv_connect_t conn; uv_tcp_t tcp; @@ -127,6 +196,8 @@ _OVER: } static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { + taosMemoryFree(cli->wbuf[0].base); + taosMemoryFree(cli->wbuf[1].base); taosMemoryFree(cli->wbuf); taosMemoryFree(cli->rbuf); taosMemoryFree(cli->addr); @@ -230,12 +301,13 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 } terrno = 0; - char header[2048] = {0}; - int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); + int32_t len = 2048; + char* header = taosMemoryCalloc(1, len); + int32_t headLen = taosBuildHttpHeader(server, contLen, header, len, flag); uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); - wb[0] = uv_buf_init((char*)header, headLen); // stack var - wb[1] = uv_buf_init((char*)pCont, contLen); // heap var + wb[0] = uv_buf_init((char*)header, strlen(header)); // heap var + wb[1] = uv_buf_init((char*)pCont, contLen); // heap var SHttpClient* cli = taosMemoryCalloc(1, sizeof(SHttpClient)); cli->conn.data = cli; @@ -281,3 +353,15 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 taosMemoryFree(loop); return terrno; } +int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { + SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); + + msg->server = strdup(server); + msg->port = port; + msg->cont = taosMemoryMalloc(contLen); + memcpy(msg->cont, pCont, contLen); + msg->flag = flag; + + transAsyncSend(http->asyncPool, &(msg->q)); + return 0; +} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 28660934f8..97915e1ded 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -187,18 +187,8 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \ } while (0) -#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \ - do { \ - if (exh == NULL) { \ - idx = -1; \ - } else { \ - ASYNC_CHECK_HANDLE((exh), refId); \ - pThrd = (SCliThrd*)(exh)->pThrd; \ - } \ - } while (0) -#define CONN_PERSIST_TIME(para) ((para) <= 90000 ? 90000 : (para)) -#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_PERSIST_TIME(para) ((para) <= 90000 ? 90000 : (para)) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) #define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ do { \ @@ -217,6 +207,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { tDebug("msg found, %" PRIu64 "", ahandle); \ } \ } while (0) + #define CONN_GET_NEXT_SENDMSG(conn) \ do { \ int i = 0; \ @@ -231,21 +222,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ } while (0) -#define CONN_HANDLE_THREAD_QUIT(thrd) \ - do { \ - if (thrd->quit) { \ - return; \ - } \ - } while (0) - -#define CONN_HANDLE_BROKEN(conn) \ - do { \ - if (conn->broken) { \ - cliHandleExcept(conn); \ - return; \ - } \ - } while (0) - #define CONN_SET_PERSIST_BY_APP(conn) \ do { \ if (conn->status == ConnNormal) { \ From 9322928a570354a54f82f184df3fdae591140d90 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 28 Oct 2022 22:41:25 +0800 Subject: [PATCH 07/60] opt http module --- source/libs/transport/src/thttp.c | 65 ++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 6715290acf..7dffb8d192 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -45,6 +45,8 @@ typedef struct SHttpMsg { static TdThreadOnce transHttpInit = PTHREAD_ONCE_INIT; static SHttpModule* http = NULL; +static void httpHandleReq(SHttpMsg* msg); + static void* httpThread(void* arg) { SHttpModule* http = (SHttpModule*)arg; setThreadName("http-cli-send-thread"); @@ -52,6 +54,13 @@ static void* httpThread(void* arg) { return NULL; } +static void httpDestroyMsg(SHttpMsg* msg) { + if (msg == NULL) return; + + taosMemoryFree(msg->server); + taosMemoryFree(msg->cont); + taosMemoryFree(msg); +} static void httpAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; SHttpModule* http = item->pThrd; @@ -68,6 +77,7 @@ static void httpAsyncCb(uv_async_t* handle) { queue* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); msg = QUEUE_DATA(h, SHttpMsg, q); + httpHandleReq(msg); } } @@ -355,7 +365,6 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 } int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); - msg->server = strdup(server); msg->port = port; msg->cont = taosMemoryMalloc(contLen); @@ -365,3 +374,57 @@ int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, i transAsyncSend(http->asyncPool, &(msg->q)); return 0; } +static void httpHandleReq(SHttpMsg* msg) { + struct sockaddr_in dest = {0}; + if (taosBuildDstAddr(msg->server, msg->port, &dest) < 0) { + httpDestroyMsg(msg); + return; + } + if (msg->flag == HTTP_GZIP) { + int32_t dstLen = taosCompressHttpRport(msg->cont, msg->len); + if (dstLen > 0) { + msg->len = dstLen; + } else { + msg->flag = HTTP_FLAT; + } + } + + terrno = 0; + + int32_t len = 2048; + char* header = taosMemoryCalloc(1, len); + int32_t headLen = taosBuildHttpHeader(msg->server, msg->len, header, len, msg->flag); + + uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); + wb[0] = uv_buf_init((char*)header, strlen(header)); // heap var + wb[1] = uv_buf_init((char*)msg->cont, msg->len); // heap var + + SHttpClient* cli = taosMemoryCalloc(1, sizeof(SHttpClient)); + cli->conn.data = cli; + cli->tcp.data = cli; + cli->req.data = cli; + cli->wbuf = wb; + cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); + cli->addr = msg->server; + cli->port = msg->port; + + taosMemoryFree(msg); + + uv_tcp_init(http->loop, &cli->tcp); + // set up timeout to avoid stuck; + int32_t fd = taosCreateSocketWithTimeout(5); + + int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + if (ret != 0) { + uError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); + destroyHttpClient(cli); + return; + } + + ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); + if (ret != 0) { + uError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, + cli->port); + destroyHttpClient(cli); + } +} From c5fba9ccbd6a38e26b653b6bf651834fdcce67de Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 28 Oct 2022 23:17:22 +0800 Subject: [PATCH 08/60] opt http module --- source/libs/transport/src/thttp.c | 74 +++---------------------------- 1 file changed, 7 insertions(+), 67 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 7dffb8d192..0c4492e1f0 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -45,7 +45,9 @@ typedef struct SHttpMsg { static TdThreadOnce transHttpInit = PTHREAD_ONCE_INIT; static SHttpModule* http = NULL; -static void httpHandleReq(SHttpMsg* msg); +static void httpHandleReq(SHttpMsg* msg); +static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag); static void* httpThread(void* arg) { SHttpModule* http = (SHttpModule*)arg; @@ -297,73 +299,11 @@ static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, return 0; } int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { - struct sockaddr_in dest = {0}; - if (taosBuildDstAddr(server, port, &dest) < 0) { - return -1; - } - if (flag == HTTP_GZIP) { - int32_t dstLen = taosCompressHttpRport(pCont, contLen); - if (dstLen > 0) { - contLen = dstLen; - } else { - flag = HTTP_FLAT; - } - } - terrno = 0; - - int32_t len = 2048; - char* header = taosMemoryCalloc(1, len); - int32_t headLen = taosBuildHttpHeader(server, contLen, header, len, flag); - - uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); - wb[0] = uv_buf_init((char*)header, strlen(header)); // heap var - wb[1] = uv_buf_init((char*)pCont, contLen); // heap var - - SHttpClient* cli = taosMemoryCalloc(1, sizeof(SHttpClient)); - cli->conn.data = cli; - cli->tcp.data = cli; - cli->req.data = cli; - cli->wbuf = wb; - cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); - cli->addr = tstrdup(server); - cli->port = port; - - uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t)); - int err = uv_loop_init(loop); - if (err != 0) { - uError("http-report failed to init uv_loop, reason: %s", uv_strerror(err)); - taosMemoryFree(loop); - terrno = TAOS_SYSTEM_ERROR(err); - destroyHttpClient(cli); - return terrno; - } - uv_tcp_init(loop, &cli->tcp); - // set up timeout to avoid stuck; - int32_t fd = taosCreateSocketWithTimeout(5); - - int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); - if (ret != 0) { - uError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); - destroyHttpClient(cli); - uv_stop(loop); - terrno = TAOS_SYSTEM_ERROR(ret); - } else { - ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); - if (ret != 0) { - uError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, - cli->port); - destroyHttpClient(cli); - uv_stop(loop); - terrno = TAOS_SYSTEM_ERROR(ret); - } - } - - uv_run(loop, UV_RUN_DEFAULT); - uv_loop_close(loop); - taosMemoryFree(loop); - return terrno; + taosThreadOnce(&transHttpInit, transHttpEnvInit); + return taosSendHttpReportImpl(server, port, pCont, contLen, flag); } -int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { +static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag) { SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); msg->server = strdup(server); msg->port = port; From 52c3777823376836714ba0570a956897c1e471a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 30 Oct 2022 03:01:40 +0800 Subject: [PATCH 09/60] refactor: do some internal refactor. --- include/common/tcommon.h | 8 - source/dnode/vnode/src/inc/vnodeInt.h | 3 - source/libs/executor/inc/executil.h | 29 ++++ source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/cachescanoperator.c | 12 +- source/libs/executor/src/executil.c | 165 ++++++++++++++++--- source/libs/executor/src/executor.c | 8 +- source/libs/executor/src/executorimpl.c | 73 ++++++-- source/libs/executor/src/scanoperator.c | 44 +++-- 9 files changed, 259 insertions(+), 85 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 2b45a5d206..3bfbb85958 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -164,14 +164,6 @@ typedef enum EStreamType { STREAM_FILL_OVER, } EStreamType; -typedef struct { - SArray* pGroupList; - SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid - bool needSortTableByGroupId; - uint64_t suid; -} STableListInfo; - #pragma pack(push, 1) typedef struct SColumnDataAgg { int16_t colId; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a5dc4431ab..94178727be 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -162,10 +162,7 @@ int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSub int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); -STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId, - void* pMemRef); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); -int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list); // tq int tqInit(); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 0cfef7dc24..9f77178966 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -97,6 +97,35 @@ typedef struct SColMatchInfo { struct SqlFunctionCtx; +// If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly +// The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups +//typedef struct STableListInfo { +// bool oneTableForEachGroup; +// int32_t numOfOuputGroups; // the data block will be generated one by one +// int32_t* groupOffset; // keep the offset value for each group in the tableList +// SArray* pTableList; +// SHashObj* map; // speedup acquire the tableQueryInfo by table uid +// uint64_t suid; +//} STableListInfo; +typedef struct { + bool oneTableForEachGroup; + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList + SArray* pGroupList; + SArray* pTableList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid + bool needSortTableByGroupId; + uint64_t suid; +} STableListInfo; + +void destroyTableList(STableListInfo* pTableList); +int32_t getNumOfOutputGroups(const STableListInfo* pTableList); +bool oneTableForEachGroup(const STableListInfo* pTableList); +int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid); +int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo, int32_t* num); +uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid); +uint64_t getTotalTables(const STableListInfo* pTableList); + size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); void closeResultRow(SResultRow* pResultRow); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8769e8ac2f..31bf410f0a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -1077,7 +1077,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); bool groupbyTbname(SNodeList* pGroupList); -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); +int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey, bool groupSort); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index f645e71c6e..6241eae3ac 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -167,17 +167,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - if (pTableList->map != NULL) { - int64_t* groupId = taosHashGet(pTableList->map, &pInfo->pRes->info.uid, sizeof(int64_t)); - if (groupId != NULL) { - pInfo->pRes->info.groupId = *groupId; - } - } else { - ASSERT(taosArrayGetSize(pTableList->pTableList) == 1); - STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, 0); - pInfo->pRes->info.groupId = pKeyInfo->groupId; - } - + pInfo->pRes->info.groupId = getTableGroupId(pTableList, pInfo->pRes->info.uid); pInfo->indexOfBufferedRes += 1; return pInfo->pRes; } else { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 971b28eb09..7448f7a8fa 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -733,7 +733,6 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis int32_t len = (int32_t)(pStart - (char*)keyBuf); info->groupId = calcGroupId(keyBuf, len); - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); } // int64_t st2 = taosGetTimestampUs(); @@ -751,14 +750,70 @@ end: return code; } +static int32_t nameComparFn(const void* p1, const void* p2) { + const char* pName1 = *(const char**)p1; + const char* pName2 = *(const char**)p2; + + int32_t ret = strcmp(pName1, pName2); + if (ret == 0) { + return 0; + } else { + return (ret > 0) ? 1 : -1; + } +} + +static SArray* getTableNameList(const SNodeListNode* pList) { + int32_t len = LIST_LENGTH(pList->pNodeList); + SListCell* cell = pList->pNodeList->pHead; + + SArray* pTbList = taosArrayInit(len, POINTER_BYTES); + for (int i = 0; i < pList->pNodeList->length; i++) { + SValueNode* valueNode = (SValueNode*)cell->pNode; + if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) { + terrno = TSDB_CODE_INVALID_PARA; + taosArrayDestroy(pTbList); + return NULL; + } + + char* name = varDataVal(valueNode->datum.p); + taosArrayPush(pTbList, &name); + cell = cell->pNext; + } + + size_t numOfTables = taosArrayGetSize(pTbList); + + // order the name + taosArraySort(pTbList, nameComparFn); + + // remove the duplicates + SArray* pNewList = taosArrayInit(taosArrayGetSize(pTbList), sizeof(void*)); + taosArrayPush(pNewList, taosArrayGet(pTbList, 0)); + + for (int32_t i = 1; i < numOfTables; ++i) { + char** name = taosArrayGetLast(pNewList); + char** nameInOldList = taosArrayGet(pTbList, i); + if (strcmp(*name, *nameInOldList) == 0) { + continue; + } + + taosArrayPush(pNewList, nameInOldList); + } + + taosArrayDestroy(pTbList); + return pNewList; +} + static int tableUidCompare(const void* a, const void* b) { - int64_t u1 = *(uint64_t*)a; - int64_t u2 = *(uint64_t*)b; + uint64_t u1 = *(uint64_t*)a; + uint64_t u2 = *(uint64_t*)b; + if (u1 == u2) { return 0; } + return u1 < u2 ? -1 : 1; } + static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond, SHashObj* tags) { int32_t ret = -1; if (nodeType(cond) == QUERY_NODE_OPERATOR) { @@ -778,7 +833,9 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list SNodeList* pList = (SNodeList*)pNode->pParameterList; int32_t len = LIST_LENGTH(pList); - if (len <= 0) return ret; + if (len <= 0) { + return ret; + } SListCell* cell = pList->pHead; for (int i = 0; i < len; i++) { @@ -789,6 +846,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list } cell = cell->pNext; } + taosArraySort(list, tableUidCompare); taosArrayRemoveDuplicate(list, tableUidCompare, NULL); @@ -796,6 +854,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list ret = metaGetTableTagsByUids(metaHandle, suid, list, tags); removeInvalidTable(list, tags); } + return ret; } @@ -831,23 +890,13 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* SNodeListNode* pList = (SNodeListNode*)pNode->pRight; int32_t len = LIST_LENGTH(pList->pNodeList); - if (len <= 0) return -1; - - SListCell* cell = pList->pNodeList->pHead; - - SArray* pTbList = taosArrayInit(len, sizeof(void*)); - for (int i = 0; i < pList->pNodeList->length; i++) { - SValueNode* valueNode = (SValueNode*)cell->pNode; - if (!IS_VAR_DATA_TYPE(valueNode->node.resType.type)) { - taosArrayDestroy(pTbList); - return -1; - } - char* name = varDataVal(valueNode->datum.p); - taosArrayPush(pTbList, &name); - cell = cell->pNext; + if (len <= 0) { + return -1; } - for (int i = 0; i < taosArrayGetSize(pTbList); i++) { + SArray* pTbList = getTableNameList(pList); + size_t num = taosArrayGetSize(pTbList); + for (int i = 0; i < num; i++) { char* name = taosArrayGetP(pTbList, i); uint64_t uid = 0; if (metaGetTableUidByName(metaHandle, name, &uid) == 0) { @@ -866,8 +915,10 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* taosArrayDestroy(pTbList); return 0; } + return -1; } + int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -1604,3 +1655,79 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimit pLimitInfo->remainOffset = limit.offset; pLimitInfo->remainGroupOffset = slimit.offset; } + +uint64_t getTotalTables(const STableListInfo* pTableList) { + ASSERT(taosArrayGetSize(pTableList->pTableList) == taosHashGetSize(pTableList->map)); + return taosArrayGetSize(pTableList->pTableList); +} + +uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { + int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); + ASSERT(pTableList->map != NULL && slot != NULL); + + STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, *slot); + ASSERT(pKeyInfo->uid == tableUid); + + return pKeyInfo->groupId; +} + +int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid) { + if (pTableList->map == NULL) { + ASSERT(taosArrayGetSize(pTableList->pTableList) == 0); + pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + } + + STableKeyInfo keyInfo = {.uid = uid, .groupId = gid}; + taosArrayPush(pTableList->pTableList, &keyInfo); + + int32_t slot = (int32_t)taosArrayGetSize(pTableList->pTableList) - 1; + taosHashPut(pTableList->map, &uid, sizeof(uid), &slot, sizeof(slot)); + + qDebug("uid:%" PRIu64 ", groupId:%" PRIu64 " added into table list, slot:%d, total:%d", uid, gid, slot, slot + 1); + return TSDB_CODE_SUCCESS; +} + +int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo, + int32_t* size) { + int32_t total = getNumOfOutputGroups(pTableList); + if (ordinalGroupIndex < 0 || ordinalGroupIndex >= total) { + return TSDB_CODE_INVALID_PARA; + } + + // here handle two special cases: + // 1. only one group exists, and 2. one table exists for each group. + if (total == 1) { + *size = getTotalTables(pTableList); + *pKeyInfo = taosArrayGet(pTableList->pTableList, 0); + return TSDB_CODE_SUCCESS; + } else if (total == getTotalTables(pTableList)) { + *size = 1; + *pKeyInfo = taosArrayGet(pTableList->pTableList, ordinalGroupIndex); + return TSDB_CODE_SUCCESS; + } + + int32_t offset = pTableList->groupOffset[ordinalGroupIndex]; + if (ordinalGroupIndex < total - 1) { + *size = pTableList->groupOffset[offset + 1] - pTableList->groupOffset[offset]; + } else { + *size = total - pTableList->groupOffset[offset] - 1; + } + + *pKeyInfo = taosArrayGet(pTableList->pTableList, offset); + return TSDB_CODE_SUCCESS; +} + +int32_t getNumOfOutputGroups(const STableListInfo* pTableList) { return pTableList->numOfOuputGroups; } + +// todo remove it +bool oneTableForEachGroup(const STableListInfo* pTableList) { return pTableList->oneTableForEachGroup; } + +void destroyTableList(STableListInfo* pTableqinfoList) { + pTableqinfoList->pTableList = taosArrayDestroy(pTableqinfoList->pTableList); + taosMemoryFreeClear(pTableqinfoList->groupOffset); + + taosHashCleanup(pTableqinfoList->map); + + pTableqinfoList->pTableList = NULL; + pTableqinfoList->map = NULL; +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 9546c3895e..12f3874e70 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -293,10 +293,6 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str); } - if (pListInfo->map == NULL) { - pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - } - // traverse to the stream scanner node to add this table id SOperatorInfo* pInfo = pTaskInfo->pRoot; while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -358,8 +354,8 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo if (!exists) { #endif - taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); - taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(*uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); + + addTableIntoTableList(&pTaskInfo->tableqinfoList, keyInfo.uid, keyInfo.groupId); } if (keyBuf != NULL) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 75db42fccc..0091b4a31d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3366,7 +3366,47 @@ static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) { static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); } + +static int32_t orderbyGroupIdComparFn(const void* p1, const void* p2) { + STableKeyInfo* pInfo1 = (STableKeyInfo*) p1; + STableKeyInfo* pInfo2 = (STableKeyInfo*) p2; + + if (pInfo1->groupId == pInfo2->groupId) { + return 0; + } else { + return pInfo1->groupId < pInfo2->groupId? -1:1; + } +} + static int32_t sortTableGroup(STableListInfo* pTableListInfo) { + int32_t code = TSDB_CODE_SUCCESS; + + taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn); + int32_t size = taosArrayGetSize(pTableListInfo->pTableList); + + SArray* pList = taosArrayInit(4, sizeof(int32_t)); + + STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0); + uint64_t gid = pInfo->groupId; + + int32_t start = 0; + taosArrayPush(pList, &start); + + for(int32_t i = 1; i < size; ++i) { + pInfo = taosArrayGet(pTableListInfo->pTableList, i); + if (pInfo->groupId != gid) { + taosArrayPush(pList, &i); + gid = pInfo->groupId; + } + } + + pTableListInfo->numOfOuputGroups = taosArrayGetSize(pList); + pTableListInfo->groupOffset = taosMemoryMalloc(sizeof(int32_t) * pTableListInfo->numOfOuputGroups); + memcpy(pTableListInfo->groupOffset, taosArrayGet(pList, 0), sizeof(int32_t) * pTableListInfo->numOfOuputGroups); + taosArrayDestroy(pList); + return TSDB_CODE_SUCCESS; + +#if 0 taosArrayClear(pTableListInfo->pGroupList); SArray* sortSupport = taosArrayInit(16, sizeof(uint64_t)); if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -3422,6 +3462,7 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { } taosArrayDestroy(sortSupport); return TDB_CODE_SUCCESS; +#endif } bool groupbyTbname(SNodeList* pGroupList) { @@ -3437,35 +3478,41 @@ bool groupbyTbname(SNodeList* pGroupList) { return bytbname; } -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group) { - if (group == NULL) { - return TDB_CODE_SUCCESS; - } - +int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) { pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (pTableListInfo->map == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - bool assignUid = groupbyTbname(group); - + bool groupByTbname = groupbyTbname(group); size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList); - if (assignUid) { + if (groupByTbname || group == NULL) { for (int32_t i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - info->groupId = info->uid; - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); + info->groupId = groupByTbname? info->uid:0; + } + + pTableListInfo->oneTableForEachGroup = groupByTbname; + + if (groupSort && groupByTbname) { + taosArraySort(pTableListInfo->pTableList, orderbyGroupIdComparFn); + pTableListInfo->numOfOuputGroups = numOfTables; } } else { int32_t code = getColInfoResultForGroupby(pHandle->meta, group, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { return code; } + + if (groupSort) { + return sortTableGroup(pTableListInfo); + } } - if (pTableListInfo->needSortTableByGroupId) { - return sortTableGroup(pTableListInfo); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &i, sizeof(int32_t)); } return TDB_CODE_SUCCESS; @@ -3551,7 +3598,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; if (pHandle->vnode) { int32_t code = - createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, + createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, /*pTableScanNode->groupSort*/false, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e2f3b1c6c4..a4a9f233ab 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -626,10 +626,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pBlock->info = binfo; ASSERT(binfo.uid != 0); - uint64_t* groupId = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); - if (groupId) { - pBlock->info.groupId = *groupId; - } + pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid); uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); @@ -683,10 +680,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; - qDebug( - "%s start to repeat ascending order scan data SELECT last_row(*),hostname from cpu group by hostname;blocks " - "due to query func required", - GET_TASKID(pTaskInfo)); + qDebug( "%s start to repeat ascending order scan data blocks due to query func required", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); @@ -755,8 +749,12 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { if (pInfo->currentGroupId == -1) { pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { - setTaskStatus(pTaskInfo, TASK_COMPLETED); + qDebug("number:------------------------%d, %d", (int)taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList), + getNumOfOutputGroups(&pTaskInfo->tableqinfoList)); + if (pInfo->currentGroupId >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)/*taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)*/) { +// if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { +// setTaskStatus(pTaskInfo, TASK_COMPLETED); + doSetOperatorCompleted(pOperator); return NULL; } @@ -790,7 +788,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return result; } - setTaskStatus(pTaskInfo, TASK_COMPLETED); + doSetOperatorCompleted(pOperator); return NULL; } @@ -1122,12 +1120,7 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, } static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { - SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map; - uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); - if (groupId) { - return *groupId; - } - return 0; + return getTableGroupId(&pInfo->pTableScanOp->pTaskInfo->tableqinfoList, uid); } static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -1549,12 +1542,13 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.version = pBlock->info.version; - uint64_t* groupIdPre = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); - if (groupIdPre) { - pInfo->pRes->info.groupId = *groupIdPre; - } else { - pInfo->pRes->info.groupId = 0; - } + pInfo->pRes->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid); +// uint64_t* groupIdPre = taosHashGet(pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); +// if (groupIdPre) { +// pInfo->pRes->info.groupId = *groupIdPre; +// } else { +// pInfo->pRes->info.groupId = 0; +// } // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { @@ -4202,6 +4196,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } + pTableListInfo->numOfOuputGroups = 1; + int64_t st1 = taosGetTimestampUs(); qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1 - st) / 1000.0, idStr); @@ -4211,7 +4207,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags } pTableListInfo->needSortTableByGroupId = groupSort; - code = generateGroupIdMap(pTableListInfo, pHandle, pGroupTags); + code = setGroupIdMapForAllTables(pTableListInfo, pHandle, pGroupTags, groupSort); if (code != TSDB_CODE_SUCCESS) { return code; } From 41c99c02efec1f0ce648952dd2ba2e6c55cace7b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 30 Oct 2022 11:17:07 +0800 Subject: [PATCH 10/60] refactor: do some internal refactor. --- source/libs/executor/src/cachescanoperator.c | 20 ++-- source/libs/executor/src/executil.c | 10 +- source/libs/executor/src/executorimpl.c | 16 +-- source/libs/executor/src/scanoperator.c | 114 +++++++++++++------ 4 files changed, 99 insertions(+), 61 deletions(-) diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 6241eae3ac..92e8d90bb8 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -62,7 +62,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pInfo->pUidList = taosArrayInit(4, sizeof(int64_t)); // partition by tbname - if (taosArrayGetSize(pTableList->pGroupList) == taosArrayGetSize(pTableList->pTableList)) { + if (oneTableForEachGroup(pTableList) || (getTotalTables(pTableList) == 1)) { pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | (pScanNode->ignoreNull ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW); code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pTableList->pTableList, @@ -175,12 +175,18 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { return NULL; } } else { - size_t totalGroups = taosArrayGetSize(pTableList->pGroupList); + size_t num = getNumOfOutputGroups(pTableList); + while (pInfo->currentGroupIndex < num) { - while (pInfo->currentGroupIndex < totalGroups) { - SArray* pGroupTableList = taosArrayGetP(pTableList->pGroupList, pInfo->currentGroupIndex); + STableKeyInfo* p = NULL; + int32_t s = 0; + getTablesOfGroup(pTableList, pInfo->currentGroupIndex, &p, &s); + SArray* x = taosArrayInit(4, sizeof(STableKeyInfo)); + for(int32_t i = 0; i < s; ++i) { + taosArrayPush(x, &p[i]); + } - tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pGroupTableList, + tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, x, taosArrayGetSize(pInfo->matchInfo.pList), pTableList->suid, &pInfo->pLastrowReader); taosArrayClear(pInfo->pUidList); @@ -195,9 +201,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { if (pInfo->pRes->info.rows > 0) { if (pInfo->pseudoExprSup.numOfExprs > 0) { SExprSupp* pSup = &pInfo->pseudoExprSup; - - STableKeyInfo* pKeyInfo = taosArrayGet(pGroupTableList, 0); - pInfo->pRes->info.groupId = pKeyInfo->groupId; + pInfo->pRes->info.groupId = p->groupId; if (taosArrayGetSize(pInfo->pUidList) > 0) { ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7448f7a8fa..c8fb0e23f8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -997,14 +997,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } taosArrayDestroy(res); - - pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); - if (pListInfo->pGroupList == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - // put into list as default group, remove it if grouping sorting is required later - taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); return code; } @@ -1698,7 +1690,7 @@ int32_t getTablesOfGroup(const STableListInfo* pTableList, int32_t ordinalGroupI // 1. only one group exists, and 2. one table exists for each group. if (total == 1) { *size = getTotalTables(pTableList); - *pKeyInfo = taosArrayGet(pTableList->pTableList, 0); + *pKeyInfo = (*size == 0)? NULL:taosArrayGet(pTableList->pTableList, 0); return TSDB_CODE_SUCCESS; } else if (total == getTotalTables(pTableList)) { *size = 1; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0091b4a31d..b0480bef35 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4026,15 +4026,15 @@ void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); if (pTableqinfoList->needSortTableByGroupId) { - for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) { - SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); - if (tmp == pTableqinfoList->pTableList) { - continue; - } - taosArrayDestroy(tmp); - } +// for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) { +// SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); +// if (tmp == pTableqinfoList->pTableList) { +// continue; +// } +// taosArrayDestroy(tmp); +// } } - taosArrayDestroy(pTableqinfoList->pGroupList); +// taosArrayDestroy(pTableqinfoList->pGroupList); pTableqinfoList->pTableList = NULL; pTableqinfoList->map = NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a4a9f233ab..b1adbd14c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -749,20 +749,28 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { if (pInfo->currentGroupId == -1) { pInfo->currentGroupId++; - qDebug("number:------------------------%d, %d", (int)taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList), - getNumOfOutputGroups(&pTaskInfo->tableqinfoList)); +// qDebug("number:------------------------%d, %d", (int)taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList), +// getNumOfOutputGroups(&pTaskInfo->tableqinfoList)); if (pInfo->currentGroupId >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)/*taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)*/) { // if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { -// setTaskStatus(pTaskInfo, TASK_COMPLETED); doSetOperatorCompleted(pOperator); return NULL; } - SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray* p = taosArrayInit(4, sizeof(STableKeyInfo)); tsdbReaderClose(pInfo->dataReader); - int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, + STableKeyInfo* x = NULL; + int32_t num = 0; + getTablesOfGroup(&pTaskInfo->tableqinfoList, pInfo->currentGroupId, &x, &num); + for(int32_t i = 0; i < num; ++i) { + taosArrayPush(p, &x[i]); + } + + int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, p, (STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); + taosArrayDestroy(p); + if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); return NULL; @@ -775,11 +783,16 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { - setTaskStatus(pTaskInfo, TASK_COMPLETED); + if (pInfo->currentGroupId >= getNumOfOutputGroups(&pTaskInfo->tableqinfoList)) { + doSetOperatorCompleted(pOperator); return NULL; } + // reset value for the next group data output + pOperator->status = OP_OPENED; + pInfo->limitInfo.numOfOutputRows = 0; + pInfo->limitInfo.remainOffset = pInfo->limitInfo.limit.offset; + tsdbReaderReset(pInfo->dataReader, &pInfo->cond); pInfo->scanTimes = 0; @@ -1075,39 +1088,59 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { pTableScanInfo->cond.twindows = *pWin; pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; -} - -static void freeArray(void* array) { taosArrayDestroy(array); } - -static void resetTableScanOperator(SOperatorInfo* pTableScanOp) { - STableScanInfo* pTableScanInfo = pTableScanOp->info; - pTableScanInfo->cond.startVersion = -1; - pTableScanInfo->cond.endVersion = -1; - SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList; - SArray* allTbls = pTableScanOp->pTaskInfo->tableqinfoList.pTableList; - taosArrayClearP(gpTbls, freeArray); - taosArrayPush(gpTbls, &allTbls); - STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - resetTableScanInfo(pTableScanOp->info, &win); + tsdbReaderClose(pTableScanInfo->dataReader); + pTableScanInfo->dataReader = NULL; } static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, int64_t maxVersion) { - SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList; - taosArrayClear(gpTbls); STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0}; - SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(tbls, &tblInfo); - taosArrayPush(gpTbls, &tbls); - STimeWindow win = {.skey = startTs, .ekey = endTs}; - STableScanInfo* pTableScanInfo = pTableScanOp->info; - pTableScanInfo->cond.startVersion = -1; - pTableScanInfo->cond.endVersion = maxVersion; - resetTableScanInfo(pTableScanOp->info, &win); - SSDataBlock* pRes = doTableScan(pTableScanOp); - resetTableScanOperator(pTableScanOp); - return pRes; + STableScanInfo* pTableScanInfo = pTableScanOp->info; + SQueryTableDataCond cond = pTableScanInfo->cond; + + cond.startVersion = -1; + cond.endVersion = maxVersion; + cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs}; + + SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; + + SSDataBlock* pBlock = pTableScanInfo->pResBlock; + blockDataCleanup(pBlock); + + SArray* p = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(p, &tblInfo); + + STsdbReader* pReader = NULL; + int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, p, (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + bool hasBlock = tsdbNextDataBlock(pReader); + if (hasBlock) { + SDataBlockInfo binfo = {0}; + tsdbRetrieveDataBlockInfo(pReader, &binfo); + + SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); + blockDataEnsureCapacity(pBlock, binfo.rows); + + pBlock->info.window = binfo.window; + pBlock->info.uid = binfo.uid; + pBlock->info.rows = binfo.rows; + + relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + + pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, binfo.uid); + } + + tsdbReaderClose(pReader); + qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 + ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); + + return pBlock->info.rows > 0 ? pBlock : NULL; } static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -2329,11 +2362,20 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pTSInfo->cond.endVersion = pHandle->version; } - SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0); +// SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, 0); + STableKeyInfo* pList = NULL; + int32_t num = 0; + getTablesOfGroup(&pTaskInfo->tableqinfoList, 0, &pList, &num); + + SArray* p = taosArrayInit(4, sizeof(STableKeyInfo)); + for(int32_t i = 0; i < num; ++i) { + taosArrayPush(p, &pList[i]); + } + if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->dataReader = NULL; - if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, tableList, &pTSInfo->dataReader, NULL) < 0) { + if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, p, &pTSInfo->dataReader, NULL) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; } From 8831f386e68fe76d1bcf3dbbd527fb0163c00e37 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 31 Oct 2022 11:57:58 +0800 Subject: [PATCH 11/60] opt http module --- source/libs/transport/src/thttp.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 0c4492e1f0..fc61fb5122 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -107,13 +107,14 @@ static void transHttpEnvDestroy() { } typedef struct SHttpClient { - uv_connect_t conn; - uv_tcp_t tcp; - uv_write_t req; - uv_buf_t* wbuf; - char* rbuf; - char* addr; - uint16_t port; + uv_connect_t conn; + uv_tcp_t tcp; + uv_write_t req; + uv_buf_t* wbuf; + char* rbuf; + char* addr; + uint16_t port; + struct sockaddr_in dest; } SHttpClient; static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, @@ -240,8 +241,7 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const static void clientSentCb(uv_write_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { - terrno = TAOS_SYSTEM_ERROR(status); - uError("http-report failed to send data %s", uv_strerror(status)); + uError("http-report failed to send data, reason: %s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } else { @@ -253,7 +253,6 @@ static void clientSentCb(uv_write_t* req, int32_t status) { } status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); if (status != 0) { - terrno = TAOS_SYSTEM_ERROR(status); uError("http-report failed to recv data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -265,7 +264,6 @@ static void clientSentCb(uv_write_t* req, int32_t status) { static void clientConnCb(uv_connect_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { - terrno = TAOS_SYSTEM_ERROR(status); uError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -276,7 +274,6 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { } status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); if (0 != status) { - terrno = TAOS_SYSTEM_ERROR(status); uError("http-report failed to send data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); @@ -289,7 +286,6 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); uError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr()); return -1; } @@ -309,6 +305,7 @@ static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* p msg->port = port; msg->cont = taosMemoryMalloc(contLen); memcpy(msg->cont, pCont, contLen); + msg->len = contLen; msg->flag = flag; transAsyncSend(http->asyncPool, &(msg->q)); @@ -329,8 +326,6 @@ static void httpHandleReq(SHttpMsg* msg) { } } - terrno = 0; - int32_t len = 2048; char* header = taosMemoryCalloc(1, len); int32_t headLen = taosBuildHttpHeader(msg->server, msg->len, header, len, msg->flag); @@ -347,6 +342,7 @@ static void httpHandleReq(SHttpMsg* msg) { cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); cli->addr = msg->server; cli->port = msg->port; + cli->dest = dest; taosMemoryFree(msg); @@ -361,7 +357,7 @@ static void httpHandleReq(SHttpMsg* msg) { return; } - ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); + ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&cli->dest, clientConnCb); if (ret != 0) { uError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); From adf085ab46a33706612ec461553f3292f032c596 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 14:17:57 +0800 Subject: [PATCH 12/60] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 20 ++++---- source/libs/executor/src/scanoperator.c | 65 +++++++++++-------------- 3 files changed, 40 insertions(+), 47 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index bed6e93e5a..3d4a3e1969 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -159,7 +159,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *pTabl void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbTableNextDataBlock(STsdbReader *pReader, int64_t uid); -void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); +void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a0fbf545f3..895b7adc5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3787,24 +3787,24 @@ bool tsdbTableNextDataBlock(STsdbReader* pReader, int64_t uid) { return true; } -static void setBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { - ASSERT(pDataBlockInfo != NULL && pReader != NULL); - pDataBlockInfo->rows = pReader->pResBlock->info.rows; - pDataBlockInfo->uid = pReader->pResBlock->info.uid; - pDataBlockInfo->window = pReader->pResBlock->info.window; +static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { + ASSERT(pReader != NULL); + *rows = pReader->pResBlock->info.rows; + *uid = pReader->pResBlock->info.uid; + *pWindow = pReader->pResBlock->info.window; } -void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { +void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_MAIN) { - setBlockInfo(pReader, pDataBlockInfo); + setBlockInfo(pReader, rows, uid, pWindow); } else if (pReader->step == EXTERNAL_ROWS_PREV) { - setBlockInfo(pReader->innerReader[0], pDataBlockInfo); + setBlockInfo(pReader->innerReader[0], rows, uid, pWindow); } else { - setBlockInfo(pReader->innerReader[1], pDataBlockInfo); + setBlockInfo(pReader->innerReader[1], rows, uid, pWindow); } } else { - setBlockInfo(pReader, pDataBlockInfo); + setBlockInfo(pReader, rows, uid, pWindow); } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b1adbd14c0..3102e63737 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -618,13 +618,14 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &binfo); + SDataBlockInfo* pBInfo = &pBlock->info; - binfo.capacity = binfo.rows; - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info = binfo; - ASSERT(binfo.uid != 0); + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &rows, &pBInfo->uid, &pBInfo->window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; + + ASSERT(pBInfo->uid != 0); pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid); @@ -1120,20 +1121,19 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU bool hasBlock = tsdbNextDataBlock(pReader); if (hasBlock) { - SDataBlockInfo binfo = {0}; - tsdbRetrieveDataBlockInfo(pReader, &binfo); + SDataBlockInfo* pBInfo = &pBlock->info; + + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->uid, &pBInfo->window); SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - blockDataEnsureCapacity(pBlock, binfo.rows); - - pBlock->info.window = binfo.window; - pBlock->info.uid = binfo.uid; - pBlock->info.rows = binfo.rows; + blockDataEnsureCapacity(pBlock, pBInfo->rows); + pBInfo->rows = rows; relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); - pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, binfo.uid); + pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBInfo->uid); } tsdbReaderClose(pReader); @@ -2121,7 +2121,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - tsdbRetrieveDataBlockInfo(pInfo->dataReader, &pBlock->info); + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.uid, &pBlock->info.window); + pBlock->info.rows = rows; SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); pBlock->pDataBlock = pCols; @@ -4561,14 +4563,11 @@ static SSDataBlock* getTableDataBlockTemp(void* param) { } blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(reader, &binfo); - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info.type = binfo.type; - pBlock->info.uid = binfo.uid; - pBlock->info.window = binfo.window; - pBlock->info.rows = binfo.rows; + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; if (tsdbIsAscendingOrder(pInfo->pReader)) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; @@ -4627,14 +4626,11 @@ static SSDataBlock* getTableDataBlock2(void* param) { } blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(reader, &binfo); - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info.type = binfo.type; - pBlock->info.uid = binfo.uid; - pBlock->info.window = binfo.window; - pBlock->info.rows = binfo.rows; + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status); @@ -4684,14 +4680,11 @@ static SSDataBlock* getTableDataBlock(void* param) { } blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(reader, &binfo); - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info.type = binfo.type; - pBlock->info.uid = binfo.uid; - pBlock->info.window = binfo.window; - pBlock->info.rows = binfo.rows; + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); From a1db402500cf88e40928df80fe495d8316c23c14 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 14:38:01 +0800 Subject: [PATCH 13/60] refactor: do some internal refactor. --- source/libs/executor/src/scanoperator.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3102e63737..2a1734b8c6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -474,6 +474,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return terrno; } + int32_t rows = pBlock->info.rows; + pBlock->info.rows = 0; + + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; + relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); @@ -619,14 +625,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { blockDataCleanup(pBlock); SDataBlockInfo* pBInfo = &pBlock->info; - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &rows, &pBInfo->uid, &pBInfo->window); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBInfo->rows, &pBInfo->uid, &pBInfo->window); +// blockDataEnsureCapacity(pBlock, rows); +// pBlock->info.rows = rows; ASSERT(pBInfo->uid != 0); - pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid); uint32_t status = 0; From 851f34ff36d0c390f58bda312b090bfb231608ae Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 31 Oct 2022 14:41:59 +0800 Subject: [PATCH 14/60] test:add replica 3 for all testcases and frameworks --- tests/pytest/util/cases.py | 4 +- tests/system-test/0-others/cachemodel.py | 2 +- tests/system-test/0-others/compatibility.py | 2 +- tests/system-test/0-others/fsync.py | 2 +- tests/system-test/0-others/show.py | 2 +- tests/system-test/0-others/sysinfo.py | 2 +- tests/system-test/0-others/taosShell.py | 2 +- tests/system-test/0-others/taosShellError.py | 2 +- tests/system-test/0-others/taosShellNetChk.py | 2 +- tests/system-test/0-others/taosdMonitor.py | 2 +- tests/system-test/0-others/taosdShell.py | 2 +- tests/system-test/0-others/taosdlog.py | 2 +- tests/system-test/0-others/telemetry.py | 2 +- tests/system-test/0-others/udfTest.py | 2 +- tests/system-test/0-others/udf_cfg1.py | 2 +- tests/system-test/0-others/udf_cfg2.py | 2 +- tests/system-test/0-others/udf_create.py | 2 +- .../system-test/0-others/udf_restart_taosd.py | 2 +- tests/system-test/0-others/user_control.py | 2 +- tests/system-test/1-insert/alter_database.py | 2 +- tests/system-test/1-insert/alter_stable.py | 2 +- tests/system-test/1-insert/alter_table.py | 2 +- tests/system-test/1-insert/block_wise.py | 2 +- .../system-test/1-insert/create_retentions.py | 2 +- .../system-test/1-insert/database_pre_suf.py | 2 +- .../system-test/1-insert/db_tb_name_check.py | 2 +- tests/system-test/1-insert/delete_data.py | 2 +- .../1-insert/influxdb_line_taosc_insert.py | 2 +- .../1-insert/insertWithMoreVgroup.py | 2 +- tests/system-test/1-insert/insert_drop.py | 2 +- tests/system-test/1-insert/keep_expired.py | 2 +- tests/system-test/1-insert/mutil_stage.py | 2 +- .../1-insert/mutipythonnodebugtaosd.py | 2 +- .../1-insert/opentsdb_json_taosc_insert.py | 2 +- .../opentsdb_telnet_line_taosc_insert.py | 2 +- tests/system-test/1-insert/table_comment.py | 2 +- tests/system-test/1-insert/table_param_ttl.py | 2 +- .../1-insert/tb_100w_data_order.py | 2 +- .../1-insert/test_stmt_muti_insert_query.py | 2 +- .../1-insert/test_stmt_set_tbname_tag.py | 2 +- tests/system-test/1-insert/time_range_wise.py | 2 +- tests/system-test/1-insert/update_data.py | 2 +- .../1-insert/update_data_muti_rows.py | 2 +- tests/system-test/2-query/Now.py | 2 +- tests/system-test/2-query/Timediff.py | 2 +- tests/system-test/2-query/To_iso8601.py | 2 +- tests/system-test/2-query/To_unixtimestamp.py | 2 +- tests/system-test/2-query/Today.py | 2 +- tests/system-test/2-query/abs.py | 2 +- tests/system-test/2-query/and_or_for_byte.py | 2 +- tests/system-test/2-query/apercentile.py | 2 +- tests/system-test/2-query/avg.py | 2 +- tests/system-test/2-query/between.py | 2 +- tests/system-test/2-query/bottom.py | 2 +- tests/system-test/2-query/cast.py | 2 +- tests/system-test/2-query/ceil.py | 2 +- tests/system-test/2-query/char_length.py | 2 +- tests/system-test/2-query/check_tsdb.py | 2 +- tests/system-test/2-query/concat.py | 2 +- tests/system-test/2-query/concat2.py | 2 +- tests/system-test/2-query/concat_ws.py | 2 +- tests/system-test/2-query/concat_ws2.py | 2 +- tests/system-test/2-query/count.py | 2 +- tests/system-test/2-query/count_partition.py | 2 +- tests/system-test/2-query/csum.py | 2 +- tests/system-test/2-query/db.py | 2 +- tests/system-test/2-query/diff.py | 2 +- tests/system-test/2-query/distinct.py | 2 +- .../2-query/distribute_agg_apercentile.py | 2 +- .../system-test/2-query/distribute_agg_avg.py | 2 +- .../2-query/distribute_agg_count.py | 2 +- .../system-test/2-query/distribute_agg_max.py | 2 +- .../system-test/2-query/distribute_agg_min.py | 2 +- .../2-query/distribute_agg_spread.py | 2 +- .../2-query/distribute_agg_stddev.py | 2 +- .../system-test/2-query/distribute_agg_sum.py | 2 +- tests/system-test/2-query/elapsed.py | 2 +- tests/system-test/2-query/explain.py | 2 +- tests/system-test/2-query/first.py | 2 +- tests/system-test/2-query/floor.py | 2 +- tests/system-test/2-query/function_diff.py | 2 +- tests/system-test/2-query/function_null.py | 2 +- .../2-query/function_stateduration.py | 2 +- tests/system-test/2-query/histogram.py | 33 ++++++++++- tests/system-test/2-query/hyperloglog.py | 2 +- tests/system-test/2-query/interp.py | 2 +- tests/system-test/2-query/irate.py | 2 +- tests/system-test/2-query/join.py | 2 +- tests/system-test/2-query/join2.py | 2 +- tests/system-test/2-query/json_tag.py | 2 +- .../2-query/json_tag_large_tables.py | 2 +- tests/system-test/2-query/last.py | 2 +- tests/system-test/2-query/last_row.py | 2 +- tests/system-test/2-query/leastsquares.py | 2 +- tests/system-test/2-query/length.py | 2 +- tests/system-test/2-query/log.py | 2 +- tests/system-test/2-query/lower.py | 2 +- tests/system-test/2-query/ltrim.py | 2 +- tests/system-test/2-query/mavg.py | 2 +- tests/system-test/2-query/max.py | 2 +- tests/system-test/2-query/max_partition.py | 2 +- tests/system-test/2-query/min.py | 2 +- tests/system-test/2-query/nestedQuery.py | 2 +- tests/system-test/2-query/percentile.py | 2 +- tests/system-test/2-query/qnodeCluster.py | 2 +- .../2-query/query_cols_tags_and_or.py | 2 +- tests/system-test/2-query/round.py | 2 +- tests/system-test/2-query/rtrim.py | 2 +- tests/system-test/2-query/sample.py | 2 +- tests/system-test/2-query/smaTest.py | 2 +- tests/system-test/2-query/sml.py | 2 +- tests/system-test/2-query/spread.py | 2 +- tests/system-test/2-query/stablity.py | 2 +- tests/system-test/2-query/statecount.py | 2 +- tests/system-test/2-query/stateduration.py | 2 +- tests/system-test/2-query/stddev.py | 2 +- tests/system-test/2-query/substr.py | 2 +- tests/system-test/2-query/sum.py | 2 +- tests/system-test/2-query/tail.py | 2 +- tests/system-test/2-query/timetruncate.py | 2 +- tests/system-test/2-query/timezone.py | 2 +- tests/system-test/2-query/top.py | 2 +- tests/system-test/2-query/tsbsQuery.py | 2 +- tests/system-test/2-query/ttl_comment.py | 2 +- tests/system-test/2-query/twa.py | 2 +- tests/system-test/2-query/union.py | 2 +- tests/system-test/2-query/union1.py | 2 +- tests/system-test/2-query/union2.py | 2 +- tests/system-test/2-query/union3.py | 2 +- tests/system-test/2-query/union4.py | 2 +- tests/system-test/2-query/unique.py | 2 +- tests/system-test/2-query/upper.py | 2 +- tests/system-test/2-query/varchar.py | 2 +- tests/system-test/5-taos-tools/TD-12478.py | 2 +- .../taosdump/taosdumpTestColTag.py | 2 +- .../5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 9 ++- ...ode3mnodeSep1VnodeStopMnodeCreateDbRep3.py | 4 +- .../6-cluster/clusterCommonCreate.py | 2 +- tests/system-test/7-tmq/basic5.py | 2 +- tests/system-test/7-tmq/create_wrong_topic.py | 2 +- .../7-tmq/dataFromTsdbNWal-multiCtb.py | 2 +- tests/system-test/7-tmq/dataFromTsdbNWal.py | 2 +- tests/system-test/7-tmq/db.py | 2 +- .../7-tmq/dropDbR3ConflictTransaction.py | 2 +- tests/system-test/7-tmq/schema.py | 2 +- tests/system-test/7-tmq/stbFilter.py | 2 +- tests/system-test/7-tmq/stbTagFilter-1ctb.py | 2 +- .../7-tmq/stbTagFilter-multiCtb.py | 2 +- tests/system-test/7-tmq/subscribeDb.py | 2 +- tests/system-test/7-tmq/subscribeDb0.py | 2 +- tests/system-test/7-tmq/subscribeDb1.py | 2 +- tests/system-test/7-tmq/subscribeDb2.py | 2 +- tests/system-test/7-tmq/subscribeDb3.py | 2 +- tests/system-test/7-tmq/subscribeDb4.py | 2 +- tests/system-test/7-tmq/subscribeStb.py | 2 +- tests/system-test/7-tmq/subscribeStb0.py | 2 +- tests/system-test/7-tmq/subscribeStb1.py | 2 +- tests/system-test/7-tmq/subscribeStb2.py | 2 +- tests/system-test/7-tmq/subscribeStb3.py | 2 +- tests/system-test/7-tmq/subscribeStb4.py | 2 +- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 2 +- tests/system-test/7-tmq/tmqAlterSchema.py | 2 +- tests/system-test/7-tmq/tmqAutoCreateTbl.py | 2 +- tests/system-test/7-tmq/tmqCheckData.py | 2 +- tests/system-test/7-tmq/tmqCheckData1.py | 2 +- tests/system-test/7-tmq/tmqCommon.py | 2 +- .../7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py | 2 +- .../system-test/7-tmq/tmqConsFromTsdb-1ctb.py | 2 +- ...nsFromTsdb-mutilVg-mutilCtb-funcNFilter.py | 2 +- .../7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py | 2 +- .../7-tmq/tmqConsFromTsdb-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb.py | 2 +- .../tmqConsFromTsdb1-1ctb-funcNFilter.py | 2 +- .../7-tmq/tmqConsFromTsdb1-1ctb.py | 2 +- ...sFromTsdb1-mutilVg-mutilCtb-funcNFilter.py | 2 +- .../tmqConsFromTsdb1-mutilVg-mutilCtb.py | 2 +- .../7-tmq/tmqConsFromTsdb1-mutilVg.py | 2 +- tests/system-test/7-tmq/tmqConsFromTsdb1.py | 2 +- tests/system-test/7-tmq/tmqConsumerGroup.py | 2 +- tests/system-test/7-tmq/tmqDelete-1ctb.py | 2 +- tests/system-test/7-tmq/tmqDelete-multiCtb.py | 2 +- tests/system-test/7-tmq/tmqDnode.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart.py | 2 +- .../system-test/7-tmq/tmqDropNtb-snapshot0.py | 2 +- .../system-test/7-tmq/tmqDropNtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqDropStb.py | 2 +- tests/system-test/7-tmq/tmqDropStbCtb.py | 2 +- tests/system-test/7-tmq/tmqError.py | 2 +- tests/system-test/7-tmq/tmqModule.py | 2 +- tests/system-test/7-tmq/tmqShow.py | 2 +- tests/system-test/7-tmq/tmqSubscribeStb-r3.py | 2 +- .../7-tmq/tmqUdf-multCtb-snapshot0.py | 2 +- .../7-tmq/tmqUdf-multCtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqUdf.py | 2 +- tests/system-test/7-tmq/tmqUpdate-1ctb.py | 2 +- .../7-tmq/tmqUpdate-multiCtb-snapshot0.py | 2 +- .../7-tmq/tmqUpdate-multiCtb-snapshot1.py | 2 +- tests/system-test/7-tmq/tmqUpdate-multiCtb.py | 2 +- .../system-test/7-tmq/tmqUpdateWithConsume.py | 2 +- tests/system-test/7-tmq/tmq_taosx.py | 2 +- tests/system-test/99-TDcase/TD-15517.py | 2 +- tests/system-test/99-TDcase/TD-15554.py | 2 +- tests/system-test/99-TDcase/TD-15557.py | 2 +- tests/system-test/99-TDcase/TD-15563.py | 2 +- tests/system-test/99-TDcase/TD-16025.py | 2 +- tests/system-test/99-TDcase/TD-16821.py | 2 +- tests/system-test/99-TDcase/TD-17255.py | 2 +- tests/system-test/99-TDcase/TD-17699.py | 2 +- tests/system-test/99-TDcase/TD-19201.py | 2 +- tests/system-test/fulltest.sh | 58 +++++++++---------- tests/system-test/test.py | 15 +++-- 211 files changed, 282 insertions(+), 251 deletions(-) mode change 100644 => 100755 tests/system-test/fulltest.sh diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 2bfd8efdcd..3eb5cf2548 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -63,14 +63,14 @@ class TDCases: tdLog.info("total %d Linux test case(s) executed" % (runNum)) - def runOneLinux(self, conn, fileName): + def runOneLinux(self, conn, fileName, replicaVar): testModule = self.__dynamicLoadModule(fileName) runNum = 0 for tmp in self.linuxCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() - case.init(conn, self._logSql) + case.init(conn, self._logSql, replicaVar) try: case.run() except Exception as e: diff --git a/tests/system-test/0-others/cachemodel.py b/tests/system-test/0-others/cachemodel.py index 42c52a2f3c..53834f792d 100644 --- a/tests/system-test/0-others/cachemodel.py +++ b/tests/system-test/0-others/cachemodel.py @@ -14,7 +14,7 @@ class TDTestCase: updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index e81579a9e4..88fbcd179d 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -22,7 +22,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index 6eefc3db24..fe470c442e 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index c25d1d1f33..6ce8872cda 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -20,7 +20,7 @@ from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/0-others/sysinfo.py b/tests/system-test/0-others/sysinfo.py index a4716dd544..b9ea39fc84 100644 --- a/tests/system-test/0-others/sysinfo.py +++ b/tests/system-test/0-others/sysinfo.py @@ -20,7 +20,7 @@ from util.common import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db' diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 879d78e7ac..accb49dbfb 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -108,7 +108,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 0dca5ff40f..e8271387a8 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -110,7 +110,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 80f6a6bc30..72c06f71be 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -110,7 +110,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 96e057c9f8..a9907419e4 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -284,7 +284,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index e09c8edce9..d09a9e7a47 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -44,7 +44,7 @@ class TDTestCase: # print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/taosdlog.py b/tests/system-test/0-others/taosdlog.py index bd3d6b9b4c..ba265abc2d 100644 --- a/tests/system-test/0-others/taosdlog.py +++ b/tests/system-test/0-others/taosdlog.py @@ -10,7 +10,7 @@ from util.dnodes import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index 812b8b40c5..021b9aed2c 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -173,7 +173,7 @@ class TDTestCase: print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index c4c40348b8..a58e22ba1c 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -13,7 +13,7 @@ import subprocess class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_cfg1.py b/tests/system-test/0-others/udf_cfg1.py index e6ab57b488..9b1874d576 100644 --- a/tests/system-test/0-others/udf_cfg1.py +++ b/tests/system-test/0-others/udf_cfg1.py @@ -15,7 +15,7 @@ class TDTestCase: updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":0} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index 3f8ba37491..67848a9430 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -15,7 +15,7 @@ class TDTestCase: updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":1} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 788202eb1b..736943cc56 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -15,7 +15,7 @@ import threading class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index b860a0dfab..02c59c21e1 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -12,7 +12,7 @@ import subprocess class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index a20b7b17bc..c6a1a7eac2 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -154,7 +154,7 @@ class User: class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py index d3a55ee0a6..6d9e04ec42 100644 --- a/tests/system-test/1-insert/alter_database.py +++ b/tests/system-test/1-insert/alter_database.py @@ -11,7 +11,7 @@ from util.cases import * from util.dnodes import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),logSql) self.buffer_boundary = [3,4097,8193,12289,16384] diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index b66cbb89c0..77ff6ae0f8 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -20,7 +20,7 @@ from util.sqlset import * from util import constant from util.common import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index 5fbfff9909..aed4089cfa 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -21,7 +21,7 @@ from util.common import * from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/1-insert/block_wise.py b/tests/system-test/1-insert/block_wise.py index 083eff34e6..53ffc0bfd6 100644 --- a/tests/system-test/1-insert/block_wise.py +++ b/tests/system-test/1-insert/block_wise.py @@ -142,7 +142,7 @@ class BSMAschema: class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.precision = "ms" diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 54abcec391..ec8f54fffa 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -44,7 +44,7 @@ NTBNAME = "nt1" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index fc4bf7f5de..a4aa8247ab 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -28,7 +28,7 @@ class TDTestCase: "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/1-insert/db_tb_name_check.py b/tests/system-test/1-insert/db_tb_name_check.py index 5017fde68a..fd0bd43b01 100644 --- a/tests/system-test/1-insert/db_tb_name_check.py +++ b/tests/system-test/1-insert/db_tb_name_check.py @@ -31,7 +31,7 @@ from util.common import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.special_name = ['!','@','#','$','%','^','&','*','(',')','[',']','{','}',\ diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index f620a4b18a..d935a08133 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -23,7 +23,7 @@ from util.common import * from util.sqlset import TDSetSql class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index cae4294bc9..82db318c2b 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -29,7 +29,7 @@ if platform.system().lower() == 'windows': sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) self._conn = conn diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index b80a5f9a4a..9d033ff6e5 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -62,7 +62,7 @@ class TDTestCase: return buildPath # init - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/insert_drop.py b/tests/system-test/1-insert/insert_drop.py index 3e30e94a6b..d063501484 100644 --- a/tests/system-test/1-insert/insert_drop.py +++ b/tests/system-test/1-insert/insert_drop.py @@ -8,7 +8,7 @@ import threading class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/1-insert/keep_expired.py b/tests/system-test/1-insert/keep_expired.py index 09db1e45ba..1a9cd1a158 100644 --- a/tests/system-test/1-insert/keep_expired.py +++ b/tests/system-test/1-insert/keep_expired.py @@ -7,7 +7,7 @@ import time class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.dbname = "test" diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 63317e8036..08f0ba8313 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -57,7 +57,7 @@ DATA_PRE2 = f"data2" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.taos_cfg_path = tdDnodes.dnodes[0].cfgPath diff --git a/tests/system-test/1-insert/mutipythonnodebugtaosd.py b/tests/system-test/1-insert/mutipythonnodebugtaosd.py index 8aea53cd92..1f568fdea1 100644 --- a/tests/system-test/1-insert/mutipythonnodebugtaosd.py +++ b/tests/system-test/1-insert/mutipythonnodebugtaosd.py @@ -60,7 +60,7 @@ class TDTestCase: return buildPath # init - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) # tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 3b01784000..9be51eb445 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -24,7 +24,7 @@ import threading import json class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self._conn = conn diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index 209cfb724e..ab11691cf8 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -28,7 +28,7 @@ if platform.system().lower() == 'windows': sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) self._conn = conn diff --git a/tests/system-test/1-insert/table_comment.py b/tests/system-test/1-insert/table_comment.py index a0ae364fcd..2a8aa9b26a 100644 --- a/tests/system-test/1-insert/table_comment.py +++ b/tests/system-test/1-insert/table_comment.py @@ -20,7 +20,7 @@ from util.sql import * from util.common import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # prepare data diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index 21c591b07e..a184c51178 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -18,7 +18,7 @@ from util.common import * class TDTestCase: updatecfgDict = {'ttlUnit':5,'ttlPushInterval':3} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ntbname = 'ntb' diff --git a/tests/system-test/1-insert/tb_100w_data_order.py b/tests/system-test/1-insert/tb_100w_data_order.py index d489ba21bc..85fe559cc2 100644 --- a/tests/system-test/1-insert/tb_100w_data_order.py +++ b/tests/system-test/1-insert/tb_100w_data_order.py @@ -5,7 +5,7 @@ from util.common import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ts = 1537146000000 diff --git a/tests/system-test/1-insert/test_stmt_muti_insert_query.py b/tests/system-test/1-insert/test_stmt_muti_insert_query.py index 7ddc0e60bd..89beb30dc9 100644 --- a/tests/system-test/1-insert/test_stmt_muti_insert_query.py +++ b/tests/system-test/1-insert/test_stmt_muti_insert_query.py @@ -57,7 +57,7 @@ class TDTestCase: return buildPath # init - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py index 321dc88cd7..535a65ffed 100644 --- a/tests/system-test/1-insert/test_stmt_set_tbname_tag.py +++ b/tests/system-test/1-insert/test_stmt_set_tbname_tag.py @@ -57,7 +57,7 @@ class TDTestCase: return buildPath # init - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index 3188583181..1499c178d4 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -134,7 +134,7 @@ class SMAschema: class TDTestCase: updatecfgDict = {"querySmaOptimize": 1} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.precision = "ms" diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 417adc39ac..e11b45190b 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -21,7 +21,7 @@ from util.sql import * from util.common import * from util.sqlset import TDSetSql class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),logSql) self.setsql = TDSetSql() diff --git a/tests/system-test/1-insert/update_data_muti_rows.py b/tests/system-test/1-insert/update_data_muti_rows.py index 623dc497ea..af78ba5d19 100644 --- a/tests/system-test/1-insert/update_data_muti_rows.py +++ b/tests/system-test/1-insert/update_data_muti_rows.py @@ -23,7 +23,7 @@ from util.common import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 35291856f2..0e514cf43b 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -7,7 +7,7 @@ from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/Timediff.py b/tests/system-test/2-query/Timediff.py index 1f73215dd5..c126ce926a 100644 --- a/tests/system-test/2-query/Timediff.py +++ b/tests/system-test/2-query/Timediff.py @@ -4,7 +4,7 @@ from util.cases import * from util.gettime import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.get_time = GetTime() diff --git a/tests/system-test/2-query/To_iso8601.py b/tests/system-test/2-query/To_iso8601.py index ccc26e9b1b..bf4fe404aa 100644 --- a/tests/system-test/2-query/To_iso8601.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -10,7 +10,7 @@ import os class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.rowNum = 10 diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 64f7b18e41..df99b08862 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -10,7 +10,7 @@ from util.sqlset import TDSetSql class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 0124c5bbac..0f89a378be 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -10,7 +10,7 @@ import pandas as pd class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.today_date = datetime.datetime.strptime(datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index c9fc025b97..6bc9457264 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -14,7 +14,7 @@ class TDTestCase: # "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, # "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/and_or_for_byte.py b/tests/system-test/2-query/and_or_for_byte.py index 7d156da379..682d729f0e 100644 --- a/tests/system-test/2-query/and_or_for_byte.py +++ b/tests/system-test/2-query/and_or_for_byte.py @@ -14,7 +14,7 @@ class TDTestCase: # "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, # "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/apercentile.py b/tests/system-test/2-query/apercentile.py index 128a03937a..8997238d7b 100644 --- a/tests/system-test/2-query/apercentile.py +++ b/tests/system-test/2-query/apercentile.py @@ -18,7 +18,7 @@ import numpy as np from util.sqlset import TDSetSql class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),False) self.rowNum = 10 diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index 5ba0a91a7e..e7086744f8 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -10,7 +10,7 @@ class TDTestCase: # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/between.py b/tests/system-test/2-query/between.py index a9dde5617d..dd7dda668b 100644 --- a/tests/system-test/2-query/between.py +++ b/tests/system-test/2-query/between.py @@ -9,7 +9,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/bottom.py b/tests/system-test/2-query/bottom.py index e75fde3ed4..4c352b619f 100644 --- a/tests/system-test/2-query/bottom.py +++ b/tests/system-test/2-query/bottom.py @@ -21,7 +21,7 @@ from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index 4045b6ad88..46a1e42158 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -12,7 +12,7 @@ from util.dnodes import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.dbname = "db" diff --git a/tests/system-test/2-query/ceil.py b/tests/system-test/2-query/ceil.py index 6777b449f9..221b571e8e 100644 --- a/tests/system-test/2-query/ceil.py +++ b/tests/system-test/2-query/ceil.py @@ -13,7 +13,7 @@ class TDTestCase: # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/char_length.py b/tests/system-test/2-query/char_length.py index c0883e665e..e8546ca72e 100644 --- a/tests/system-test/2-query/char_length.py +++ b/tests/system-test/2-query/char_length.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [TS_COL] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/check_tsdb.py b/tests/system-test/2-query/check_tsdb.py index f504a55dcd..0cbbc60171 100644 --- a/tests/system-test/2-query/check_tsdb.py +++ b/tests/system-test/2-query/check_tsdb.py @@ -12,7 +12,7 @@ class TDTestCase: # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index 23b964012a..dc05a17185 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/concat2.py b/tests/system-test/2-query/concat2.py index 5442220076..1316a3a228 100644 --- a/tests/system-test/2-query/concat2.py +++ b/tests/system-test/2-query/concat2.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index ad784d92ec..8443d41818 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/concat_ws2.py b/tests/system-test/2-query/concat_ws2.py index caaae6cecb..38b93660e1 100644 --- a/tests/system-test/2-query/concat_ws2.py +++ b/tests/system-test/2-query/concat_ws2.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 4d2a1cf07c..6362974d0b 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -3,7 +3,7 @@ from util.sql import * from util.cases import * from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),False) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py index 90a6d9225b..4ab51c4efe 100644 --- a/tests/system-test/2-query/count_partition.py +++ b/tests/system-test/2-query/count_partition.py @@ -4,7 +4,7 @@ from util.sql import * from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/csum.py b/tests/system-test/2-query/csum.py index 83aaca8f12..fb828b85df 100644 --- a/tests/system-test/2-query/csum.py +++ b/tests/system-test/2-query/csum.py @@ -26,7 +26,7 @@ from util.dnodes import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index f2d85ebf65..aea16dd162 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -11,7 +11,7 @@ import random class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index ceef3f93eb..6ec17ec49a 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -6,7 +6,7 @@ import numpy as np class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index 7214caec96..c7c6e1c9b0 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -9,7 +9,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index 9b8ec2ece8..d2364df65a 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -8,7 +8,7 @@ import random class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_avg.py b/tests/system-test/2-query/distribute_agg_avg.py index c54b89a2ba..4b5d3d8c5a 100644 --- a/tests/system-test/2-query/distribute_agg_avg.py +++ b/tests/system-test/2-query/distribute_agg_avg.py @@ -9,7 +9,7 @@ import platform class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index f61b6d7c2e..e6cf718c05 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -8,7 +8,7 @@ import random class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index 51b0ad4f94..92188bd765 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -9,7 +9,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index 543142ddfa..afc42755b3 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -9,7 +9,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py index 222bd7315e..ab073c76a2 100644 --- a/tests/system-test/2-query/distribute_agg_spread.py +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -8,7 +8,7 @@ import random class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to execute {__file__}") tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_stddev.py b/tests/system-test/2-query/distribute_agg_stddev.py index 6b77fc7368..ae9479dcdd 100644 --- a/tests/system-test/2-query/distribute_agg_stddev.py +++ b/tests/system-test/2-query/distribute_agg_stddev.py @@ -9,7 +9,7 @@ import math class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/distribute_agg_sum.py b/tests/system-test/2-query/distribute_agg_sum.py index 9be4602fd9..59804b7dc5 100644 --- a/tests/system-test/2-query/distribute_agg_sum.py +++ b/tests/system-test/2-query/distribute_agg_sum.py @@ -9,7 +9,7 @@ import platform class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/elapsed.py b/tests/system-test/2-query/elapsed.py index 333c60286e..03de3c35ed 100644 --- a/tests/system-test/2-query/elapsed.py +++ b/tests/system-test/2-query/elapsed.py @@ -20,7 +20,7 @@ from util.sql import * from util.dnodes import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/explain.py b/tests/system-test/2-query/explain.py index 21e16fab43..1126e23f06 100644 --- a/tests/system-test/2-query/explain.py +++ b/tests/system-test/2-query/explain.py @@ -30,7 +30,7 @@ ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_C DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/first.py b/tests/system-test/2-query/first.py index 1aa51f8bb7..9523527076 100644 --- a/tests/system-test/2-query/first.py +++ b/tests/system-test/2-query/first.py @@ -23,7 +23,7 @@ import numpy as np class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/floor.py b/tests/system-test/2-query/floor.py index 793a56c1e5..65974ff2e9 100644 --- a/tests/system-test/2-query/floor.py +++ b/tests/system-test/2-query/floor.py @@ -12,7 +12,7 @@ DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/function_diff.py b/tests/system-test/2-query/function_diff.py index 946453bb23..7f11b12e89 100644 --- a/tests/system-test/2-query/function_diff.py +++ b/tests/system-test/2-query/function_diff.py @@ -26,7 +26,7 @@ from util.dnodes import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/function_null.py b/tests/system-test/2-query/function_null.py index 93b46d2da1..3a74ca268f 100644 --- a/tests/system-test/2-query/function_null.py +++ b/tests/system-test/2-query/function_null.py @@ -11,7 +11,7 @@ import random class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/function_stateduration.py b/tests/system-test/2-query/function_stateduration.py index fec09e786b..8f25595713 100644 --- a/tests/system-test/2-query/function_stateduration.py +++ b/tests/system-test/2-query/function_stateduration.py @@ -5,7 +5,7 @@ from util.cases import * DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/histogram.py b/tests/system-test/2-query/histogram.py index 1dc5bdfa21..2a63fd6d8e 100644 --- a/tests/system-test/2-query/histogram.py +++ b/tests/system-test/2-query/histogram.py @@ -142,7 +142,7 @@ class Hsgschema: class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) @@ -304,7 +304,9 @@ class TDTestCase: err_sqls.append( Hsgschema( col=INT_COL, bin_type="USER_INPUT", user_input="[0,3,6,9,'a']", normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="USER_INPUT", user_input="['a']", normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin=['{"start": 1, "width": 3, "count": 10, "infinity": false}'], normalized=1 ) ) - err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='[{"start": 1, "width": 3, "count": 10, "infinity": false}]', normalized=1 ) ) + # err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='{"start": 1, "width": 2, "count": 10, "infinity": false}', normalized=1 , "123" ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='{"start": 1, "width": float("inf"), "count": 10, "infinity": false}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='{"start": float("inf"), "width": 10, "count": 10, "infinity": false}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='{"begin": 1, "width": 3, "count": 10, "infinity": false}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='{"start": 1, "length": 3, "count": 10, "infinity": false}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="linear_bin", linear_bin='{"start": 1, "width": 3, "num": 10, "infinity": false}', normalized=1 ) ) @@ -322,6 +324,11 @@ class TDTestCase: err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin=['{"start": 1, "factor": 4, "count": 4, "infinity": true}'], normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='[{"start": 1, "factor": 4, "count": 4, "infinity": true}]', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"begin": 1, "factor": 4, "count": 4, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": -10, "count": 4, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 0, "count": 4, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 0, "count": 4, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": -10, "width": NULL, "count": 4, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "step": 4, "count": 4, "infinity": true}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 4, "num": 4, "infinity": true}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 4, "count": 4, "witgnull": true}', normalized=1 ) ) @@ -335,6 +342,13 @@ class TDTestCase: err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": -10, "count": 4, "infinity": true}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": 0, "infinity": true}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": -10, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": 10001, "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": "123", "infinity": true}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": float("inf"), "infinity": false}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": float("inf"), "count": 10, "infinity": false}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": float("inf"), "factor": 10, "count": 10, "infinity": false}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": float("-inf"), "factor": 10, "count": 10, "infinity": false}', normalized=1 ) ) + err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": 10, "infinity": "true"}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": 10, "infinity": null}', normalized=1 ) ) err_sqls.append( Hsgschema( col=INT_COL, bin_type="log_bin", log_bin='{"start": 1, "factor": 10, "count": 10, "infinity": false}', where_clause=f"count({INT_COL}) >= 0 ") ) @@ -404,6 +418,21 @@ class TDTestCase: tdSql.checkData(0, 0, '{"lower_bin":0, "upper_bin":3, "count":3}') tdSql.checkData(1, 0, '{"lower_bin":3, "upper_bin":6, "count":3}') tdSql.checkData(2, 0, '{"lower_bin":6, "upper_bin":9, "count":3}') + + #error sql + # tdSql.error(f"SELECT HISTOGRAM(c_int, 'linear_bin', '{"start": -200, "width": 100, "count": 20, "infinity": false}', 0 ,1 ) from {dbname}.nt1 where c_int < 10") + + # tdSql.error(f"SELECT HISTOGRAM(c_int, 'linear_bin', '{"start": -200, "width": 100, "count": -10, "infinity": false}', 0 ) from {dbname}.nt1 where c_int < 10") + + # tdSql.error(f"SELECT HISTOGRAM(c_int, 'linear_bin', '{"start": -200, "width": 100, "count": 1001, "infinity": false}', 0 ) from {dbname}.nt1 where c_int < 10") + + tdSql.error(f"SELECT HISTOGRAM(c_int, 'log_bin', '[0,3,6,9]', 0 ,1 ) from {dbname}.nt1 where c_int < 10") + + tdSql.error(f"SELECT HISTOGRAM(c_int, 'USER_INPUT', '[0,3,6,9]', 0 ,1 ) from {dbname}.nt1 where c_int < 10") + tdSql.error(f"SELECT HISTOGRAM(c_int, 'USER_INPUT', '[0,3,6,9]' ) from {dbname}.nt1 where c_int < 10") + tdSql.error(f"SELECT HISTOGRAM('123', 'USER_INPUT', '[0,3,6,9]', 0 ) from {dbname}.nt1 where c_int < 10") + tdSql.error(f"SELECT HISTOGRAM('123', 123 ,123 ,123 ) from {dbname}.nt1 where c_int < 10") + tdSql.error(f"SELECT HISTOGRAM(123, '123' ,'123' ,'123' ) from {dbname}.nt1 where c_int < 10") def all_test(self, dbname=DBNAME): self.test_histogram(dbname) diff --git a/tests/system-test/2-query/hyperloglog.py b/tests/system-test/2-query/hyperloglog.py index 68f7ebdf2e..eff687cba7 100644 --- a/tests/system-test/2-query/hyperloglog.py +++ b/tests/system-test/2-query/hyperloglog.py @@ -31,7 +31,7 @@ class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 3a05583418..afc8c06bde 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -9,7 +9,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index 408f4b3749..7b3ec10a8d 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -11,7 +11,7 @@ import random ,math class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) self.tb_nums = 10 diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 9d30e1946a..04b4fa89a2 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -63,7 +63,7 @@ class DataSet: class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/join2.py b/tests/system-test/2-query/join2.py index 5c8fe0f0f9..79cc62e4bc 100644 --- a/tests/system-test/2-query/join2.py +++ b/tests/system-test/2-query/join2.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index d9715579ae..c6f1e5076d 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -13,7 +13,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py index 9164c108f9..06170e0adf 100644 --- a/tests/system-test/2-query/json_tag_large_tables.py +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -30,7 +30,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): self.testcasePath = os.path.split(__file__)[0] self.testcaseFilename = os.path.split(__file__)[-1] # os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index 3bca9f1671..0d721c8360 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -8,7 +8,7 @@ import numpy as np class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 6286852641..333c1516b3 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -11,7 +11,7 @@ import random class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) self.tb_nums = 10 diff --git a/tests/system-test/2-query/leastsquares.py b/tests/system-test/2-query/leastsquares.py index fe7188a545..8870e416f8 100644 --- a/tests/system-test/2-query/leastsquares.py +++ b/tests/system-test/2-query/leastsquares.py @@ -29,7 +29,7 @@ ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_C DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/length.py b/tests/system-test/2-query/length.py index 1761572245..44c951dd4a 100644 --- a/tests/system-test/2-query/length.py +++ b/tests/system-test/2-query/length.py @@ -23,7 +23,7 @@ DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index 358d2b9551..791065924c 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -11,7 +11,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/lower.py b/tests/system-test/2-query/lower.py index 0e33e3834e..f8ac1ab217 100644 --- a/tests/system-test/2-query/lower.py +++ b/tests/system-test/2-query/lower.py @@ -22,7 +22,7 @@ TS_TYPE_COL = [TS_COL] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/ltrim.py b/tests/system-test/2-query/ltrim.py index 330f688990..ba769ba350 100644 --- a/tests/system-test/2-query/ltrim.py +++ b/tests/system-test/2-query/ltrim.py @@ -27,7 +27,7 @@ DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index b52217af9a..7f545f2048 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -27,7 +27,7 @@ from util.dnodes import * dbname = 'db' class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index 5cc9a2d9e8..80e7fa7f04 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -6,7 +6,7 @@ import numpy as np class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 08bb7675ad..f50b9fb2bb 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -4,7 +4,7 @@ from util.sql import * from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py index d97c4340f4..8849b14a33 100644 --- a/tests/system-test/2-query/min.py +++ b/tests/system-test/2-query/min.py @@ -6,7 +6,7 @@ import numpy as np class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index c6567a8e3b..4fbd840efe 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -28,7 +28,7 @@ class TDTestCase: "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/percentile.py b/tests/system-test/2-query/percentile.py index 3b027ed6a0..58660eec6a 100644 --- a/tests/system-test/2-query/percentile.py +++ b/tests/system-test/2-query/percentile.py @@ -21,7 +21,7 @@ from util.sqlset import TDSetSql class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/qnodeCluster.py b/tests/system-test/2-query/qnodeCluster.py index 9e49bff938..6f3856146d 100644 --- a/tests/system-test/2-query/qnodeCluster.py +++ b/tests/system-test/2-query/qnodeCluster.py @@ -26,7 +26,7 @@ class TDTestCase: updatecfgDict = {'keepColumnName': 1} updatecfgDict["clientCfg"] = clientCfgDict - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index af3fbb83c0..9fc016cb40 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -16,7 +16,7 @@ from util.sql import tdSql from util.common import tdCom import random class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): ## add for TD-6672 tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/round.py b/tests/system-test/2-query/round.py index 1d69d3c9af..8f969a71d7 100644 --- a/tests/system-test/2-query/round.py +++ b/tests/system-test/2-query/round.py @@ -9,7 +9,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/rtrim.py b/tests/system-test/2-query/rtrim.py index 80307e8534..e4835fbd8e 100644 --- a/tests/system-test/2-query/rtrim.py +++ b/tests/system-test/2-query/rtrim.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index 7f1d7ab8c0..d1890d8623 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -24,7 +24,7 @@ DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ts = 1537146000000 diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 0217b6c28c..66a89d8701 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -31,7 +31,7 @@ class TDTestCase: # updatecfgDict = {'fqdn': 135} # init - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) tdSql.prepare() diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 4dae2ad6c0..64fe92438c 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/2-query/spread.py b/tests/system-test/2-query/spread.py index ffe86ff363..79881d7293 100644 --- a/tests/system-test/2-query/spread.py +++ b/tests/system-test/2-query/spread.py @@ -30,7 +30,7 @@ DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/stablity.py b/tests/system-test/2-query/stablity.py index b434335970..3c8d18e9e6 100755 --- a/tests/system-test/2-query/stablity.py +++ b/tests/system-test/2-query/stablity.py @@ -28,7 +28,7 @@ class TDTestCase: "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/2-query/statecount.py b/tests/system-test/2-query/statecount.py index c73c955de4..1d9a04cc73 100644 --- a/tests/system-test/2-query/statecount.py +++ b/tests/system-test/2-query/statecount.py @@ -12,7 +12,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.ts = 1420041600000 # 2015-01-01 00:00:00 this is begin time for first record diff --git a/tests/system-test/2-query/stateduration.py b/tests/system-test/2-query/stateduration.py index b6ddff4017..009ec37764 100644 --- a/tests/system-test/2-query/stateduration.py +++ b/tests/system-test/2-query/stateduration.py @@ -17,7 +17,7 @@ from util.sql import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.ts = 1537146000000 diff --git a/tests/system-test/2-query/stddev.py b/tests/system-test/2-query/stddev.py index 837353dfc9..d044fb0dd7 100644 --- a/tests/system-test/2-query/stddev.py +++ b/tests/system-test/2-query/stddev.py @@ -20,7 +20,7 @@ from util.sql import * from util.common import * from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.dbname = 'db_test' diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index ea55c5e44e..9b317aeab6 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -29,7 +29,7 @@ LENS = 6 class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(),False) diff --git a/tests/system-test/2-query/sum.py b/tests/system-test/2-query/sum.py index dbc79e25f5..ec1f60a720 100644 --- a/tests/system-test/2-query/sum.py +++ b/tests/system-test/2-query/sum.py @@ -24,7 +24,7 @@ DBNAME = "db" class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/tail.py b/tests/system-test/2-query/tail.py index 687023f57e..3d5ecfaa9a 100644 --- a/tests/system-test/2-query/tail.py +++ b/tests/system-test/2-query/tail.py @@ -11,7 +11,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index 4cc6b27d64..917b98daa9 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -7,7 +7,7 @@ import time from datetime import datetime from util.gettime import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) print(conn) diff --git a/tests/system-test/2-query/timezone.py b/tests/system-test/2-query/timezone.py index 9a4953909c..0a2da114de 100644 --- a/tests/system-test/2-query/timezone.py +++ b/tests/system-test/2-query/timezone.py @@ -11,7 +11,7 @@ if platform.system().lower() == 'windows': class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/top.py b/tests/system-test/2-query/top.py index 7fd143f683..1cba18c2e2 100644 --- a/tests/system-test/2-query/top.py +++ b/tests/system-test/2-query/top.py @@ -19,7 +19,7 @@ from util.sql import * from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.setsql = TDSetSql() diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 04a80a74ad..c4604799a1 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -20,7 +20,7 @@ class TDTestCase: updatecfgDict = {'keepColumnName': 1} updatecfgDict["clientCfg"] = clientCfgDict - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index 187df2e2d3..fabd641da7 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/2-query/twa.py b/tests/system-test/2-query/twa.py index a499c17efb..1fd3b8fdda 100644 --- a/tests/system-test/2-query/twa.py +++ b/tests/system-test/2-query/twa.py @@ -9,7 +9,7 @@ import math class TDTestCase: updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) self.vnode_disbutes = None diff --git a/tests/system-test/2-query/union.py b/tests/system-test/2-query/union.py index 4040bb71cb..737817f262 100644 --- a/tests/system-test/2-query/union.py +++ b/tests/system-test/2-query/union.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union1.py b/tests/system-test/2-query/union1.py index ea6940246e..1ca25e7844 100644 --- a/tests/system-test/2-query/union1.py +++ b/tests/system-test/2-query/union1.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union2.py b/tests/system-test/2-query/union2.py index 2d5e2f70bf..c063b5c383 100644 --- a/tests/system-test/2-query/union2.py +++ b/tests/system-test/2-query/union2.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union3.py b/tests/system-test/2-query/union3.py index 30a15e7624..3322f30359 100644 --- a/tests/system-test/2-query/union3.py +++ b/tests/system-test/2-query/union3.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/union4.py b/tests/system-test/2-query/union4.py index 4b2fba4272..2da83d8190 100644 --- a/tests/system-test/2-query/union4.py +++ b/tests/system-test/2-query/union4.py @@ -26,7 +26,7 @@ TS_TYPE_COL = [ TS_COL, ] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index ec77cbbcdc..cf23328fa3 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -12,7 +12,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/upper.py b/tests/system-test/2-query/upper.py index f15a6f3ba7..cc5347774c 100644 --- a/tests/system-test/2-query/upper.py +++ b/tests/system-test/2-query/upper.py @@ -22,7 +22,7 @@ TS_TYPE_COL = [TS_COL] class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/varchar.py b/tests/system-test/2-query/varchar.py index 17c3ea6333..3d5f443b3c 100644 --- a/tests/system-test/2-query/varchar.py +++ b/tests/system-test/2-query/varchar.py @@ -9,7 +9,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/5-taos-tools/TD-12478.py b/tests/system-test/5-taos-tools/TD-12478.py index 69849d3c7a..576e59f339 100644 --- a/tests/system-test/5-taos-tools/TD-12478.py +++ b/tests/system-test/5-taos-tools/TD-12478.py @@ -36,7 +36,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py b/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py index 8c62f713b8..8cb58b8d70 100644 --- a/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py +++ b/tests/system-test/5-taos-tools/taosdump/taosdumpTestColTag.py @@ -28,7 +28,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index 58ed8cde63..575e2ef712 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -25,13 +25,12 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self,conn ,logSql,replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - - + self.replicaVar = replicaVar def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -88,7 +87,7 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allDbNumbers=(paraDict['dbNumbers']*restartNumbers) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) - + paraDict['replica']=int(self.replicaVar) tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) @@ -172,7 +171,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=5,stopRole='mnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='mnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py index 3dc3e7ec65..d24c749515 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self,conn ,logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) @@ -170,7 +170,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=3,stopRole='mnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=3,stopRole='mnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 299829144e..6c2017f91c 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -33,7 +33,7 @@ from util.common import * # INSERT_DATA = 3 class ClusterComCreate: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdSql.init(conn.cursor()) # tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index 9531541f13..d1985c6567 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -26,7 +26,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py index d18cb0260e..dbdcedcbe8 100644 --- a/tests/system-test/7-tmq/create_wrong_topic.py +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -13,7 +13,7 @@ from util.dnodes import * from util.sqlset import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(),logSql) self.setsql = TDSetSql() diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py index 4add73ec2b..484969010c 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal-multiCtb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/dataFromTsdbNWal.py b/tests/system-test/7-tmq/dataFromTsdbNWal.py index 950c8fdcf6..ed22df2b07 100644 --- a/tests/system-test/7-tmq/dataFromTsdbNWal.py +++ b/tests/system-test/7-tmq/dataFromTsdbNWal.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py index da5d7fefd2..17668d8d03 100644 --- a/tests/system-test/7-tmq/db.py +++ b/tests/system-test/7-tmq/db.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index fc4fdcecf9..af6c51d947 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -26,7 +26,7 @@ class TDTestCase: self.ctbNum = 2 self.rowsPerTbl = 2 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 34d36e5792..2a67cd1bc1 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py index 4942a39db4..0f0e7c5287 100644 --- a/tests/system-test/7-tmq/stbFilter.py +++ b/tests/system-test/7-tmq/stbFilter.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/stbTagFilter-1ctb.py b/tests/system-test/7-tmq/stbTagFilter-1ctb.py index 6cb152342b..0ef454f3e2 100644 --- a/tests/system-test/7-tmq/stbTagFilter-1ctb.py +++ b/tests/system-test/7-tmq/stbTagFilter-1ctb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py index 9053bf2620..0ec114d3bd 100644 --- a/tests/system-test/7-tmq/stbTagFilter-multiCtb.py +++ b/tests/system-test/7-tmq/stbTagFilter-multiCtb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index ba46f72695..f738f816c7 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index 7720001fbb..81d824156d 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index 404938158f..02e72d0475 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb2.py b/tests/system-test/7-tmq/subscribeDb2.py index 4702aef035..edf1a8ec80 100644 --- a/tests/system-test/7-tmq/subscribeDb2.py +++ b/tests/system-test/7-tmq/subscribeDb2.py @@ -21,7 +21,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 0ff609ab6b..34df61dc09 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index d253c34a53..27efbee016 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -51,7 +51,7 @@ class TDTestCase: hostname = socket.gethostname() - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") logSql = False tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index 2757e590a3..213f9aa89c 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index 26e834ae2c..0c188754b2 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index 0c49636b15..4d5407e927 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index 2cbb16ec00..bb38a981d3 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 9c1b3fd241..32272491c6 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index 33f4c4af1a..e347d27ac1 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 7ba914c05c..837317e5d7 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -32,7 +32,7 @@ class TDTestCase: self.portStep = 100 self.dnodeOfLeader = 0 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py index 232a1e11fa..3bff0e4754 100644 --- a/tests/system-test/7-tmq/tmqAlterSchema.py +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -32,7 +32,7 @@ class TDTestCase: self.portStep = 100 self.dnodeOfLeader = 0 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index ce2cad4b14..568e49388b 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 500 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index 9338debfa6..9995af15c2 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index 7c236bbe8b..5b055cf725 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index b1455ebe48..e71e4d257d 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -37,7 +37,7 @@ from util.common import * # INSERT_DATA = 3 class TMQCom: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdSql.init(conn.cursor()) # tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py index 20b0c65c71..655097c924 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 100000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py index 494952ecd5..709464fc1e 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-1ctb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 100000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py index 666ca6ca64..09ad6e38c9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 3000 self.rowsPerTbl = 150 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py index e4ce3b0f77..95e060d581 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 3000 self.rowsPerTbl = 70 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py index da7d9e4651..fc3437613b 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 10 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py index b3bb5f84e4..73af4d196f 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 10 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py index 07fb9c7751..20db44ef19 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 1000000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py index 313ff7476e..bd42f74c03 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-1ctb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 100000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index 05f7030169..2eb1a7c52a 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 4000 self.rowsPerTbl = 150 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py index 232d90848f..351a10ed1f 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 3000 self.rowsPerTbl = 70 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py index 5841c6d605..6504274993 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 10 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1.py b/tests/system-test/7-tmq/tmqConsFromTsdb1.py index d0ab8d4fe3..2cfb6581d8 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 10 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index b5cbfb8a51..5c7d21069b 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 8329b00145..7509e43af7 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py index c9d90b45be..ded6fe94d7 100644 --- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py +++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDnode.py b/tests/system-test/7-tmq/tmqDnode.py index 802993e924..921e543e2d 100644 --- a/tests/system-test/7-tmq/tmqDnode.py +++ b/tests/system-test/7-tmq/tmqDnode.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index bcc6725848..cb7e3152cb 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py index b952dc2d57..0d75df58f9 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot0.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1000 self.rowsPerTbl = 10 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py index 4cb208b616..79cb79a83c 100644 --- a/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqDropNtb-snapshot1.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1000 self.rowsPerTbl = 10 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqDropStb.py b/tests/system-test/7-tmq/tmqDropStb.py index b172224c2a..7679f0ca44 100644 --- a/tests/system-test/7-tmq/tmqDropStb.py +++ b/tests/system-test/7-tmq/tmqDropStb.py @@ -51,7 +51,7 @@ class TDTestCase: hostname = socket.gethostname() - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") logSql = False tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index 704811d083..95780538d1 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 2d7b464025..757a3611f2 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqModule.py b/tests/system-test/7-tmq/tmqModule.py index 1ff47da159..0063edce56 100644 --- a/tests/system-test/7-tmq/tmqModule.py +++ b/tests/system-test/7-tmq/tmqModule.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index 0691da6786..6ff08498ea 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index c5f98bc3a0..33b8268a88 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -27,7 +27,7 @@ class TDTestCase: self.ctbNum = 1000 self.rowsPerTbl = 100 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index f70c6ad49b..8158249fa6 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -23,7 +23,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index 58cdc775ee..97e50cf654 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -23,7 +23,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 5eac0ae69d..d081423142 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -23,7 +23,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index 12de04cd9c..a95920b7c4 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 1 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py index 02641d8bcb..0ced8fc34b 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot0.py @@ -22,7 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 self.autoCtbPrefix = 'aCtb' - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py index baeb70e656..bdcbb9578e 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb-snapshot1.py @@ -22,7 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 self.autoCtbPrefix = 'aCtb' - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py index bde266e634..df150130c0 100644 --- a/tests/system-test/7-tmq/tmqUpdate-multiCtb.py +++ b/tests/system-test/7-tmq/tmqUpdate-multiCtb.py @@ -22,7 +22,7 @@ class TDTestCase: self.rowsPerTbl = 1000 self.autoCtbPrefix = 'aCtb' - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqUpdateWithConsume.py b/tests/system-test/7-tmq/tmqUpdateWithConsume.py index be07ba13a9..dbeac427ac 100644 --- a/tests/system-test/7-tmq/tmqUpdateWithConsume.py +++ b/tests/system-test/7-tmq/tmqUpdateWithConsume.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 626e935733..fd7e1b7bb1 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py index fe2d9b9041..e45a54c272 100644 --- a/tests/system-test/99-TDcase/TD-15517.py +++ b/tests/system-test/99-TDcase/TD-15517.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15554.py b/tests/system-test/99-TDcase/TD-15554.py index 16a34086b7..02654b5703 100644 --- a/tests/system-test/99-TDcase/TD-15554.py +++ b/tests/system-test/99-TDcase/TD-15554.py @@ -19,7 +19,7 @@ class TDTestCase: #updatecfgDict["clientCfg"] = clientCfgDict #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15557.py b/tests/system-test/99-TDcase/TD-15557.py index 243a0a4d7e..ca29e1282f 100644 --- a/tests/system-test/99-TDcase/TD-15557.py +++ b/tests/system-test/99-TDcase/TD-15557.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-15563.py b/tests/system-test/99-TDcase/TD-15563.py index 74b5472d89..6b37f25bb6 100644 --- a/tests/system-test/99-TDcase/TD-15563.py +++ b/tests/system-test/99-TDcase/TD-15563.py @@ -20,7 +20,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-16025.py b/tests/system-test/99-TDcase/TD-16025.py index 4cf94f5daf..0adc313429 100644 --- a/tests/system-test/99-TDcase/TD-16025.py +++ b/tests/system-test/99-TDcase/TD-16025.py @@ -27,7 +27,7 @@ class TDTestCase: #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal #print ("===================: ", updatecfgDict) - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index 31c1c34485..b2b5e3f425 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -15,7 +15,7 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/99-TDcase/TD-17255.py b/tests/system-test/99-TDcase/TD-17255.py index 5d18fee8d2..bcf02b654f 100644 --- a/tests/system-test/99-TDcase/TD-17255.py +++ b/tests/system-test/99-TDcase/TD-17255.py @@ -20,7 +20,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 10000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/99-TDcase/TD-17699.py b/tests/system-test/99-TDcase/TD-17699.py index 40650e4b92..d4c4a4cc32 100644 --- a/tests/system-test/99-TDcase/TD-17699.py +++ b/tests/system-test/99-TDcase/TD-17699.py @@ -52,7 +52,7 @@ class TDTestCase: hostname = socket.gethostname() - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") logSql = False tdSql.init(conn.cursor(), logSql) diff --git a/tests/system-test/99-TDcase/TD-19201.py b/tests/system-test/99-TDcase/TD-19201.py index 26caa27815..0fd86bf6b4 100644 --- a/tests/system-test/99-TDcase/TD-19201.py +++ b/tests/system-test/99-TDcase/TD-19201.py @@ -13,7 +13,7 @@ from util.dnodes import * class TDTestCase: hostname = socket.gethostname() - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh old mode 100644 new mode 100755 index fd2ff916fd..12a20821f8 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,35 +2,35 @@ set -e set -x -python3 ./test.py -f 0-others/taosShell.py -python3 ./test.py -f 0-others/taosShellError.py -python3 ./test.py -f 0-others/taosShellNetChk.py -python3 ./test.py -f 0-others/telemetry.py -python3 ./test.py -f 0-others/taosdMonitor.py -python3 ./test.py -f 0-others/udfTest.py -python3 ./test.py -f 0-others/udf_create.py -python3 ./test.py -f 0-others/udf_restart_taosd.py -python3 ./test.py -f 0-others/cachemodel.py -python3 ./test.py -f 0-others/udf_cfg1.py -python3 ./test.py -f 0-others/udf_cfg2.py -python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 -python3 ./test.py -f 0-others/sysinfo.py -python3 ./test.py -f 0-others/user_control.py -python3 ./test.py -f 0-others/fsync.py -python3 ./test.py -f 0-others/compatibility.py -python3 ./test.py -f 1-insert/alter_database.py -python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -python3 ./test.py -f 1-insert/alter_stable.py -python3 ./test.py -f 1-insert/alter_table.py -python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -python3 ./test.py -f 1-insert/table_comment.py -python3 ./test.py -f 1-insert/time_range_wise.py -python3 ./test.py -f 1-insert/block_wise.py -python3 ./test.py -f 1-insert/create_retentions.py +# python3 ./test.py -f 0-others/taosShell.py +# python3 ./test.py -f 0-others/taosShellError.py +# python3 ./test.py -f 0-others/taosShellNetChk.py +# python3 ./test.py -f 0-others/telemetry.py +# python3 ./test.py -f 0-others/taosdMonitor.py +# python3 ./test.py -f 0-others/udfTest.py +# python3 ./test.py -f 0-others/udf_create.py +# python3 ./test.py -f 0-others/udf_restart_taosd.py +# python3 ./test.py -f 0-others/cachemodel.py +# python3 ./test.py -f 0-others/udf_cfg1.py +# python3 ./test.py -f 0-others/udf_cfg2.py +# python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +# python3 ./test.py -f 0-others/sysinfo.py +# python3 ./test.py -f 0-others/user_control.py +# python3 ./test.py -f 0-others/fsync.py +# python3 ./test.py -f 0-others/compatibility.py +# python3 ./test.py -f 1-insert/alter_database.py +# python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +# python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +# python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +# python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +# python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +# python3 ./test.py -f 1-insert/alter_stable.py +# python3 ./test.py -f 1-insert/alter_table.py +# python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +# python3 ./test.py -f 1-insert/table_comment.py +# python3 ./test.py -f 1-insert/time_range_wise.py +# python3 ./test.py -f 1-insert/block_wise.py +# python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py python3 ./test.py -f 1-insert/mutil_stage.py python3 ./test.py -f 1-insert/table_param_ttl.py -R diff --git a/tests/system-test/test.py b/tests/system-test/test.py index dd49f037bd..b25bda4a3b 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -72,8 +72,9 @@ if __name__ == "__main__": queryPolicy = 1 createDnodeNums = 1 restful = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate']) + replicaVar = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -95,8 +96,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-D taosadapter update cfg dict ') - - + tdLog.printNoPrefix('-n the number of replicas') sys.exit(0) if key in ['-r', '--restart']: @@ -168,6 +168,9 @@ if __name__ == "__main__": print('adapter cfg update convert fail.') sys.exit(0) + if key in ['-n', '--replicaVar']: + replicaVar = value + if not execCmd == "": if restful: tAdapter.init(deployPath) @@ -512,7 +515,7 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - tdCases.runOneLinux(conn, fileName) + tdCases.runOneLinux(conn, fileName, replicaVar) if restart: if fileName == "all": @@ -529,7 +532,7 @@ if __name__ == "__main__": conn = taosrest.connect(url=f"http://{host}:6041") tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") - tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py", replicaVar) else: tdLog.info("not need to query") From 7135f35b21903a3209552b8f97332740c9d70184 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 15:06:51 +0800 Subject: [PATCH 15/60] refactor: do some internal refactor. --- source/libs/executor/src/executil.c | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/scanoperator.c | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c8fb0e23f8..6d6e765574 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1666,7 +1666,7 @@ uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { int32_t addTableIntoTableList(STableListInfo* pTableList, uint64_t uid, uint64_t gid) { if (pTableList->map == NULL) { ASSERT(taosArrayGetSize(pTableList->pTableList) == 0); - pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pTableList->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); } STableKeyInfo keyInfo = {.uid = uid, .groupId = gid}; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b0480bef35..7a3454e092 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3479,7 +3479,7 @@ bool groupbyTbname(SNodeList* pGroupList) { } int32_t setGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) { - pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (pTableListInfo->map == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2a1734b8c6..2d3521b419 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -626,8 +626,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { SDataBlockInfo* pBInfo = &pBlock->info; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBInfo->rows, &pBInfo->uid, &pBInfo->window); -// blockDataEnsureCapacity(pBlock, rows); -// pBlock->info.rows = rows; ASSERT(pBInfo->uid != 0); pBlock->info.groupId = getTableGroupId(&pTaskInfo->tableqinfoList, pBlock->info.uid); From 63aa6f7a471ca2d3112a3589c1f80915ae4a641f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 31 Oct 2022 15:20:20 +0800 Subject: [PATCH 16/60] fix(query): leastsquares handles unsigned types TD-19891 --- source/libs/function/src/builtinsimpl.c | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5dcef7cd17..df6ec5064d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2194,6 +2194,56 @@ int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { break; } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t* plist = (uint8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t* plist = (uint16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_UINT: { + uint32_t* plist = (uint32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t* plist = (uint64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { float* plist = (float*)pCol->pData; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { From ba37a581929300835f012d57555b2e1d87686f7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 15:39:16 +0800 Subject: [PATCH 17/60] enh(query): reduce the number of malloc. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 35 +++++++++++++------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 23615fb8af..d6a91d049a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1212,25 +1212,24 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } -static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, - int32_t* nextIndex, int32_t order) { +static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, + int32_t* nextIndex, int32_t order, SDataBlk* pBlock) { bool asc = ASCENDING_TRAVERSE(order); if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { - return NULL; + return false; } if (!asc && pBlockInfo->tbBlockIdx == 0) { - return NULL; + return false; } int32_t step = asc ? 1 : -1; *nextIndex = pBlockInfo->tbBlockIdx + step; - SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk)); - int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); + int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk); - return pBlock; + return true; } static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { @@ -1364,13 +1363,14 @@ typedef struct { static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, STsdbReader* pReader) { - int32_t neighborIndex = 0; - SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order); + int32_t neighborIndex = 0; + SDataBlk block = {0}; + + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order, &block); // overlap with neighbor - if (pNeighbor) { - pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); - taosMemoryFree(pNeighbor); + if (hasNeighbor) { + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, &block, pReader->order); } // has duplicated ts of different version in this block @@ -3067,15 +3067,14 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t nextIndex = -1; - SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); - if (pNeighborBlock == NULL) { // do nothing + int32_t nextIndex = -1; + SDataBlk block = {0}; + bool hasNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order, &block); + if (!hasNeighbor) { // do nothing return 0; } - bool overlap = overlapWithNeighborBlock(pBlock, pNeighborBlock, pReader->order); - taosMemoryFree(pNeighborBlock); - + bool overlap = overlapWithNeighborBlock(pBlock, &block, pReader->order); if (overlap) { // load next block SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; From 6b744129948c2ccd356548b5848083d92df853b8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 31 Oct 2022 16:30:42 +0800 Subject: [PATCH 18/60] refactor(sync): print fatal log when commit error --- source/libs/sync/src/syncMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 02e5c643a4..82ae8d7e51 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -3380,6 +3380,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde // ASSERT(pEntry != NULL); if (code != 0 || pEntry == NULL) { syncNodeErrorLog(ths, "get log entry error"); + sFatal("vgId:%d, get log entry %" PRId64 " error when commit since %s", ths->vgId, i, terrstr()); continue; } } From aa2b54c57c11fcf37cccb269b370f5e43041e0ee Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 31 Oct 2022 16:45:39 +0800 Subject: [PATCH 19/60] enhance: add configuration for udfd LD_LIBRARY_PATH to taos.cfg --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 6 ++++-- source/libs/function/src/tudf.c | 21 ++++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index cb4426f8a9..7df6124152 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -119,6 +119,7 @@ extern SDiskCfg tsDiskCfg[]; // udf extern bool tsStartUdfd; extern char tsUdfdResFuncs[]; +extern char tsUdfdLdLibPath[]; // schemaless extern char tsSmlChildTableName[]; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fbb9e04a25..660730f0dc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -163,7 +163,8 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds -char tsUdfdResFuncs[1024] = ""; // udfd resident funcs that teardown when udfd exits +char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits +char tsUdfdLdLibPath[512] = ""; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -424,6 +425,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; + if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1; GRANT_CFG_ADD; return 0; } @@ -722,7 +724,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); - + tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath)); if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ea59e92e98..eff3c6cc7b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -117,10 +117,29 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char dnodeIdEnvItem[32] = {0}; char thrdPoolSizeEnvItem[32] = {0}; snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId); + float numCpuCores = 4; taosGetCpuCores(&numCpuCores); snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2); - char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL}; + + char pathTaosdLdLib[512] = {0}; + size_t taosdLdLibPathLen = sizeof(pathTaosdLdLib); + uv_os_getenv("LD_LIBRARY_PATH", pathTaosdLdLib, &taosdLdLibPathLen); + + char udfdPathLdLib[1024] = {0}; + size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); + strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); + udfdPathLdLib[udfdLdLibPathLen] = ':'; + strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen); + if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { + fnInfo("udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); + } else { + fnError("can not set correct udfd LD_LIBRARY_PATH"); + } + char ldLibPathEnvItem[1024 + 32] = {0}; + snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", ldLibPathEnvItem); + + char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem, NULL}; options.env = envUdfd; int err = uv_spawn(&pData->loop, &pData->process, &options); From 9703ab510ba66357bae66bd52cc9364bd943dc3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 17:19:15 +0800 Subject: [PATCH 20/60] enh(query): avoid parse the mapdata to improve the performance by keep the offset of each block. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 41 ++++++++++++-------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index d6a91d049a..7c2c4be5b2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -35,11 +35,16 @@ typedef struct { int32_t numOfLastFiles; } SBlockNumber; +typedef struct SBlockIndex { + int32_t ordinalIndex; + int64_t inFileoffset; +} SBlockIndex; + typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; SMapData mapData; // block info (compressed) - SArray* pBlockList; // block data index list + SArray* pBlockList; // block data index list, SArray SIterInfo iter; // mem buffer skip list iterator SIterInfo iiter; // imem buffer skip list iterator SArray* delSkyline; // delete info for this table @@ -568,7 +573,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, STableBlockScanInfo* pScanInfo = p; if (pScanInfo->pBlockList == NULL) { - pScanInfo->pBlockList = taosArrayInit(4, sizeof(int32_t)); + pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBlockIndex)); } taosArrayPush(pIndexList, pBlockIdx); @@ -630,7 +635,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN continue; } - void* p = taosArrayPush(pScanInfo->pBlockList, &j); + void* p = taosArrayPush(pScanInfo->pBlockList, + &(SBlockIndex){.ordinalIndex = j, .inFileoffset = block.aSubBlock->offset}); if (p == NULL) { tMapDataClear(&pScanInfo->mapData); return TSDB_CODE_OUT_OF_MEMORY; @@ -1059,8 +1065,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) return TSDB_CODE_INVALID_PARA; } - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk); + SBlockIndex* pIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); + tMapDataGetItemByIdx(&pScanInfo->mapData, pIndex->ordinalIndex, &pBlockIter->block, tGetDataBlk); } #if 0 @@ -1073,6 +1079,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { bool asc = ASCENDING_TRAVERSE(pReader->order); + SBlockOrderSupporter sup = {0}; pBlockIter->numOfBlocks = numOfBlocks; taosArrayClear(pBlockIter->blockList); pBlockIter->pTableMap = pReader->status.pTableMap; @@ -1081,9 +1088,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); int64_t st = taosGetTimestampUs(); - - SBlockOrderSupporter sup = {0}; - int32_t code = initBlockOrderSupporter(&sup, numOfTables); + int32_t code = initBlockOrderSupporter(&sup, numOfTables); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1111,17 +1116,11 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; - SDataBlk block = {0}; + for (int32_t k = 0; k < num; ++k) { - SBlockOrderWrapper wrapper = {0}; - - int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k); - tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk); - - wrapper.uid = pTableScanInfo->uid; - wrapper.offset = block.aSubBlock[0].offset; - - sup.pDataBlockInfo[sup.numOfTables][k] = wrapper; + SBlockIndex* pIndex = taosArrayGet(pTableScanInfo->pBlockList, k); + sup.pDataBlockInfo[sup.numOfTables][k] = + (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pIndex->inFileoffset}; cnt++; } @@ -1226,9 +1225,8 @@ static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBl int32_t step = asc ? 1 : -1; *nextIndex = pBlockInfo->tbBlockIdx + step; - int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); - - tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk); + SBlockIndex* pIndex = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); + tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); return true; } @@ -3495,7 +3493,6 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order); -// resetDataBlockScanInfo(pReader->status.pTableMap, pReader->window.skey); // no data in files, let's try buffer in memory if (pReader->status.fileIter.numOfFiles == 0) { From fac516a196ba89c7081258903b7a9b11dc59f8ed Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 17:20:27 +0800 Subject: [PATCH 21/60] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 7c2c4be5b2..8d877dea7b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -635,8 +635,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN continue; } - void* p = taosArrayPush(pScanInfo->pBlockList, - &(SBlockIndex){.ordinalIndex = j, .inFileoffset = block.aSubBlock->offset}); + SBlockIndex bIndex = {.ordinalIndex = j, .inFileoffset = block.aSubBlock->offset}; + void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex); if (p == NULL) { tMapDataClear(&pScanInfo->mapData); return TSDB_CODE_OUT_OF_MEMORY; From 1fa5eafa8ace99eb956097670eccbdab056e8fde Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 17:49:50 +0800 Subject: [PATCH 22/60] ehn(query): keep the time window for each block to avoid repeatly parsing the compressed data block meta info. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 41 ++++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8d877dea7b..b454b25ea7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -36,8 +36,9 @@ typedef struct { } SBlockNumber; typedef struct SBlockIndex { - int32_t ordinalIndex; - int64_t inFileoffset; + int32_t ordinalIndex; + int64_t inFileOffset; + STimeWindow window; } SBlockIndex; typedef struct STableBlockScanInfo { @@ -635,7 +636,9 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN continue; } - SBlockIndex bIndex = {.ordinalIndex = j, .inFileoffset = block.aSubBlock->offset}; + SBlockIndex bIndex = {.ordinalIndex = j, .inFileOffset = block.aSubBlock->offset}; + bIndex.window = (STimeWindow) {.skey = block.minKey.ts, .ekey = block.maxKey.ts}; + void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex); if (p == NULL) { tMapDataClear(&pScanInfo->mapData); @@ -1120,7 +1123,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte for (int32_t k = 0; k < num; ++k) { SBlockIndex* pIndex = taosArrayGet(pTableScanInfo->pBlockList, k); sup.pDataBlockInfo[sup.numOfTables][k] = - (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pIndex->inFileoffset}; + (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pIndex->inFileOffset}; cnt++; } @@ -1212,7 +1215,7 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p } static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, - int32_t* nextIndex, int32_t order, SDataBlk* pBlock) { + int32_t* nextIndex, int32_t order, SBlockIndex* pBlockIndex) { bool asc = ASCENDING_TRAVERSE(order); if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { return false; @@ -1224,9 +1227,8 @@ static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBl int32_t step = asc ? 1 : -1; *nextIndex = pBlockInfo->tbBlockIdx + step; - - SBlockIndex* pIndex = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); - tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); + *pBlockIndex = *(SBlockIndex*) taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); +// tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); return true; } @@ -1269,12 +1271,12 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t return TSDB_CODE_SUCCESS; } -static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) { +static bool overlapWithNeighborBlock(SDataBlk* pBlock, SBlockIndex* pNeighborBlockIndex, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { - return pBlock->maxKey.ts == pNeighbor->minKey.ts; + return pBlock->maxKey.ts == pNeighborBlockIndex->window.skey; } else { - return pBlock->minKey.ts == pNeighbor->maxKey.ts; + return pBlock->minKey.ts == pNeighborBlockIndex->window.ekey; } } @@ -1361,14 +1363,14 @@ typedef struct { static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, STsdbReader* pReader) { - int32_t neighborIndex = 0; - SDataBlk block = {0}; + int32_t neighborIndex = 0; + SBlockIndex bIndex = {0}; - bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order, &block); + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order, &bIndex); // overlap with neighbor if (hasNeighbor) { - pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, &block, pReader->order); + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, &bIndex, pReader->order); } // has duplicated ts of different version in this block @@ -3065,14 +3067,15 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t nextIndex = -1; - SDataBlk block = {0}; - bool hasNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order, &block); + int32_t nextIndex = -1; + SBlockIndex bIndex = {0}; + + bool hasNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order, &bIndex); if (!hasNeighbor) { // do nothing return 0; } - bool overlap = overlapWithNeighborBlock(pBlock, &block, pReader->order); + bool overlap = overlapWithNeighborBlock(pBlock, &bIndex, pReader->order); if (overlap) { // load next block SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; From f29127a15ec24b558b2677d84a00e67d471583b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 18:04:58 +0800 Subject: [PATCH 23/60] ehn(query): do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 +- source/libs/executor/src/executorimpl.c | 51 ------------------------- source/libs/executor/src/scanoperator.c | 3 +- 3 files changed, 5 insertions(+), 53 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b454b25ea7..2b3eff0297 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3868,7 +3868,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); int32_t i = 0, j = 0; - while (j < numOfCols && i < taosArrayGetSize(pSup->pColAgg)) { + size_t size = taosArrayGetSize(pSup->pColAgg); + + while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); if (pAgg->colId == pSup->colIds[j]) { if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 11dbfaff9f..5ceda8bd78 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -844,57 +844,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int return win; } -#if 0 -static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { - STimeWindow w = {0}; - - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - - if (true) { - // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); - assert(w.ekey >= pBlockInfo->window.skey); - - if (w.ekey < pBlockInfo->window.ekey) { - return true; - } - - while (1) { - // getNextTimeWindow(pQueryAttr, &w); - if (w.skey > pBlockInfo->window.ekey) { - break; - } - - assert(w.ekey > pBlockInfo->window.ekey); - if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { - return true; - } - } - } else { - // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); - assert(w.skey <= pBlockInfo->window.ekey); - - if (w.skey > pBlockInfo->window.skey) { - return true; - } - - while (1) { - // getNextTimeWindow(pQueryAttr, &w); - if (w.ekey < pBlockInfo->window.skey) { - break; - } - - assert(w.skey < pBlockInfo->window.skey); - if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { - return true; - } - } - } - - return false; -} -#endif - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 855be4752d..d499091ccb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -331,7 +331,8 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, } } - for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->matchInfo.pList); ++i) { + size_t num = taosArrayGetSize(pTableScanInfo->matchInfo.pList); + for (int32_t i = 0; i < num; ++i) { SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i); if (!pColMatchInfo->needOutput) { continue; From d246514da28d742614cbffdf45623b2e017bb650 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 31 Oct 2022 18:16:58 +0800 Subject: [PATCH 24/60] refactor(sync): check msgcb, putToQueueFp NULL --- source/dnode/mnode/impl/src/mndSync.c | 37 ++++++++++++++++++++++++-- source/dnode/vnode/src/vnd/vnodeSync.c | 16 +++++++++-- source/libs/sync/src/syncMain.c | 8 +++--- source/libs/sync/src/syncTimeout.c | 2 +- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index df4b526775..6853ff5ccc 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,11 +17,44 @@ #include "mndSync.h" #include "mndTrans.h" -static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { +static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + SMsgHead *pHead = pMsg->pCont; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return -1; + } + + int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} + +static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + SMsgHead *pHead = pMsg->pCont; + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return -1; + } + int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); @@ -212,7 +245,7 @@ int32_t mndInitSync(SMnode *pMnode) { .msgcb = NULL, .FpSendMsg = mndSyncSendMsg, .FpEqMsg = mndSyncEqMsg, - .FpEqCtrlMsg = NULL, + .FpEqCtrlMsg = mndSyncEqCtrlMsg, }; snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 7acf5b4003..4af89dfa38 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -365,7 +365,13 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { - if (msgcb == NULL) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; return -1; } @@ -378,7 +384,13 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { } static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { - if (msgcb == NULL) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; return -1; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 82ae8d7e51..b4d939e980 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -508,7 +508,7 @@ int32_t syncEndSnapshot(int64_t rid) { SSyncLogStoreData* pData = pSyncNode->pLogStore->data; code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, wal snapshot end error since:%s", pSyncNode->vgId, terrstr(terrno)); + sError("vgId:%d, wal snapshot end error since:%s", pSyncNode->vgId, terrstr()); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return -1; @@ -2793,7 +2793,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - if (pSyncNode->FpEqMsg != NULL) { + if (pSyncNode->FpEqMsg != NULL && pSyncNode->msgcb != NULL && pSyncNode->msgcb->putToQueueFp != NULL) { int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { sError("vgId:%d, sync enqueue elect msg error, code:%d", pSyncNode->vgId, code); @@ -3379,8 +3379,8 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde // ASSERT(code == 0); // ASSERT(pEntry != NULL); if (code != 0 || pEntry == NULL) { - syncNodeErrorLog(ths, "get log entry error"); - sFatal("vgId:%d, get log entry %" PRId64 " error when commit since %s", ths->vgId, i, terrstr()); + syncNodeErrorLog(ths, "get log entry error"); + sFatal("vgId:%d, get log entry %" PRId64 " error when commit since %s", ths->vgId, i, terrstr()); continue; } } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 5ff73a6406..1c08217099 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -76,7 +76,7 @@ int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncLogStoreData* pData = ths->pLogStore->data; int32_t code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, wal snapshot end error since:%s", ths->vgId, terrstr(terrno)); + sError("vgId:%d, timer wal snapshot end error since:%s", ths->vgId, terrstr()); return -1; } else { do { From b8e451d272057480cefd74ca629743790b468678 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 31 Oct 2022 18:33:30 +0800 Subject: [PATCH 25/60] fix(stream): scalar stream update data --- source/libs/executor/src/groupoperator.c | 18 +-- source/libs/executor/src/scanoperator.c | 121 +++++++++--------- source/libs/executor/src/timewindowoperator.c | 5 +- tests/script/tsim/stream/scalar.sim | 94 ++++++++++++++ 4 files changed, 167 insertions(+), 71 deletions(-) create mode 100644 tests/script/tsim/stream/scalar.sim diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 644f02f60d..d8340b72f2 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -62,7 +62,7 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { - SColumn* pCol = (SColumn*) taosArrayGet(pGroupColList, i); + SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); (*keyLen) += pCol->bytes; // actual data + null_flag SGroupKeys key = {0}; @@ -397,7 +397,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { return buildGroupResultDataBlock(pOperator); } -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode *pAggNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) { SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -442,8 +442,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, - destroyGroupOperatorInfo, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -765,7 +765,6 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); @@ -819,8 +818,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, - NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -965,6 +964,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { case STREAM_DELETE_DATA: { copyDataBlock(pInfo->pDelRes, pBlock); pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; + printDataBlock(pInfo->pDelRes, "stream partitionby delete"); return pInfo->pDelRes; } break; default: @@ -1014,6 +1014,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->partitionSup = *pParSup; pScanInfo->pPartScalarSup = pExpr; + if (!pScanInfo->pUpdateInfo) { + pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0); + } } SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, @@ -1108,7 +1111,6 @@ _error: return NULL; } - SArray* extractColumnInfo(SNodeList* pNodeList) { size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b9324d0c64..582c79e8d3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -363,7 +363,8 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo if (pLimitInfo->remainOffset >= pBlock->info.rows) { pLimitInfo->remainOffset -= pBlock->info.rows; pBlock->info.rows = 0; - qDebug("current block ignore due to offset, current:%"PRId64", %s", pLimitInfo->remainOffset, GET_TASKID(pTaskInfo)); + qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, + GET_TASKID(pTaskInfo)); } else { blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); pLimitInfo->remainOffset = 0; @@ -376,7 +377,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo int32_t keep = pBlock->info.rows - overflowRows; blockDataKeepFirstNRows(pBlock, keep); - qDebug("output limit %"PRId64" has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo)); + qDebug("output limit %" PRId64 " has reached, %s", pLimit->limit, GET_TASKID(pTaskInfo)); pOperator->status = OP_EXEC_DONE; } } @@ -748,13 +749,13 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - int32_t num = 0; + int32_t num = 0; STableKeyInfo* pList = NULL; tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->dataReader == NULL); - int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, pList, num, (STsdbReader**)&pInfo->dataReader, - GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, pList, num, + (STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -776,7 +777,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->limitInfo.numOfOutputRows = 0; pInfo->limitInfo.remainOffset = pInfo->limitInfo.limit.offset; - int32_t num = 0; + int32_t num = 0; STableKeyInfo* pList = NULL; tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); @@ -819,8 +820,8 @@ static void destroyTableScanOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -884,7 +885,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->cost.openCost = 0; return pOperator; - _error: +_error: if (pInfo != NULL) { destroyTableScanOperatorInfo(pInfo); } @@ -1024,7 +1025,8 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC return TSDB_CODE_SUCCESS; } -SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, + SExecTaskInfo* pTaskInfo) { SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1041,8 +1043,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi } STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); cleanupQueryTableDataCond(&cond); @@ -1070,7 +1072,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; - _error: +_error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; @@ -1135,8 +1137,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU blockDataCleanup(pBlock); STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, - GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, + GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -1162,7 +1164,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU tsdbReaderClose(pReader); qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 - ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); + ", suid:%" PRIu64, + pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); return pBlock->info.rows > 0 ? pBlock : NULL; } @@ -1178,12 +1181,12 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { return getTableGroupId(pInfo->pTableScanOp->pTaskInfo->pTableInfoList, uid); -// SHashObj* map = pInfo->pTableScanOp->pTaskInfo->pTableInfoList.map; -// uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); -// if (groupId) { -// return *groupId; -// } -// return 0; + // SHashObj* map = pInfo->pTableScanOp->pTaskInfo->pTableInfoList.map; + // uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); + // if (groupId) { + // return *groupId; + // } + // return 0; } static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -1380,7 +1383,7 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS if (rows == 0) { return TSDB_CODE_SUCCESS; } - int32_t code = blockDataEnsureCapacity(pDestBlock, rows * 2); + int32_t code = blockDataEnsureCapacity(pDestBlock, rows); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1423,39 +1426,33 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS } static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { - if (pSrcBlock->info.rows == 0) { + blockDataCleanup(pDestBlock); + int32_t rows = pSrcBlock->info.rows; + if (rows == 0) { return TSDB_CODE_SUCCESS; } - blockDataCleanup(pDestBlock); - int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows); + int32_t code = blockDataEnsureCapacity(pDestBlock, rows); if (code != TSDB_CODE_SUCCESS) { return code; } - ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3); - SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* startData = (TSKEY*)pStartTsCol->pData; - SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* endData = (TSKEY*)pEndTsCol->pData; - SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* uidCol = (uint64_t*)pUidCol->pData; - SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); - SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); - SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); - SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); - SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int32_t dummy = 0; - int64_t version = pSrcBlock->info.version - 1; + SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; + SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; + ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; + int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { - uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); - colDataAppend(pDestStartCol, i, (const char*)(startData + i), false); - colDataAppend(pDestEndCol, i, (const char*)(endData + i), false); - colDataAppendNULL(pDestUidCol, i); - colDataAppend(pDestGpCol, i, (const char*)&groupId, false); - colDataAppendNULL(pDestCalStartTsCol, i); - colDataAppendNULL(pDestCalEndTsCol, i); - pDestBlock->info.rows++; + uint64_t srcUid = srcUidData[i]; + uint64_t groupId = srcGp[i]; + if (groupId == 0) { + groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + } + appendOneRowToStreamSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId, NULL); } return TSDB_CODE_SUCCESS; } @@ -1466,6 +1463,8 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock); } else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) { code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock); + } else { + code = generateDeleteResultBlock(pInfo, pSrcBlock, pDestBlock); } pDestBlock->info.type = STREAM_CLEAR; pDestBlock->info.version = pSrcBlock->info.version; @@ -1894,8 +1893,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { #endif size_t total = taosArrayGetSize(pInfo->pBlockLists); - // TODO: refactor - FETCH_NEXT_BLOCK: +// TODO: refactor +FETCH_NEXT_BLOCK: if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { doClearBufferedBlocks(pInfo); @@ -2022,7 +2021,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { int32_t totBlockNum = taosArrayGetSize(pInfo->pBlockLists); - NEXT_SUBMIT_BLK: + NEXT_SUBMIT_BLK: while (1) { if (pInfo->tqReader->pMsg == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { @@ -2278,7 +2277,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; - _end: +_end: taosMemoryFree(pInfo); taosMemoryFree(pOperator); pTaskInfo->code = code; @@ -2388,7 +2387,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } STableKeyInfo* pList = NULL; - int32_t num = 0; + int32_t num = 0; tableListGetGroupList(pTaskInfo->pTableInfoList, 0, &pList, &num); if (pHandle->initTableReader) { @@ -2467,7 +2466,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys return pOperator; - _error: +_error: if (pColIds != NULL) { taosArrayDestroy(pColIds); } @@ -4102,7 +4101,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan return pOperator; - _error: +_error: taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -4241,7 +4240,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi return pOperator; - _error: +_error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -4264,8 +4263,8 @@ int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx, STsdbReader** ppReader, const char* idstr) { STsdbReader* pReader = NULL; - void* pStart = tableListGetInfo(pTableListInfo, tableStartIdx); - int32_t num = tableEndIdx - tableStartIdx + 1; + void* pStart = tableListGetInfo(pTableListInfo, tableStartIdx); + int32_t num = tableEndIdx - tableStartIdx + 1; int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr); if (code != 0) { @@ -4426,7 +4425,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; -// STsdbReader* reader = pTableScanInfo->pReader; // taosArrayGetP(pTableScanInfo->dataReaders, readerIdx); + // STsdbReader* reader = pTableScanInfo->pReader; // taosArrayGetP(pTableScanInfo->dataReaders, readerIdx); if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -4524,7 +4523,7 @@ static SSDataBlock* getTableDataBlockTemp(void* param) { int64_t st = taosGetTimestampUs(); - void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex); + void* p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->readHandle; tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo)); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 49a7112eba..ee45d0547f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1687,7 +1687,9 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.pIntervalAggSup = pSup; - pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark); + if (!pScanInfo->pUpdateInfo) { + pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, pTwSup->waterMark); + } pScanInfo->interval = *pInterval; pScanInfo->twAggSup = *pTwSup; } @@ -2453,7 +2455,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } else { // non-linear interpolation pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - } } diff --git a/tests/script/tsim/stream/scalar.sim b/tests/script/tsim/stream/scalar.sim new file mode 100644 index 0000000000..e211357276 --- /dev/null +++ b/tests/script/tsim/stream/scalar.sim @@ -0,0 +1,94 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create stable st(ts timestamp,a int,b int,c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams0 into streamt0 as select ts c1, a, abs(b) c4 from t1 partition by a; +sql create stream streams1 into streamt1 as select ts c1, a, abs(b) c4 from t1; +sql create stream streams2 into streamt2 as select ts c1, a, abs(b) c4 from st partition by tbname; + +sql insert into t1 values(1648791213000,1,1,1,1); +sql insert into t1 values(1648791213001,1,1,1,1); +sql insert into t1 values(1648791213002,1,1,1,1); + +sql insert into t2 values(1648791213000,1,2,2,2); +sql insert into t2 values(1648791213001,1,1,1,1); +sql insert into t2 values(1648791213002,1,1,1,1); + +sql insert into t1 values(1648791213001,2,11,11,11); + + +$loop_count = 0 +loop1: + +sleep 200 + +sql select * from streamt0 order by a desc; + +$loop_count = $loop_count + 1 + +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print ======streamt0=rows=$rows + goto loop1 +endi + +if $data01 != 2 then + print ======streamt0=data01=$data01 + goto loop1 +endi + +if $data02 != 11 then + print ======streamt0=data02=$data02 + goto loop1 +endi + + +sql select * from streamt1 order by a desc; + +if $rows != 3 then + print ======streamt1=rows=$rows + goto loop1 +endi + +if $data01 != 2 then + print ======streamt1=data01=$data01 + goto loop1 +endi + +if $data02 != 11 then + print ======streamt1=data02=$data02 + goto loop1 +endi + +sql select * from streamt2 order by a desc; + +if $rows != 6 then + print ======streamt2=rows=$rows + goto loop1 +endi + +if $data01 != 2 then + print ======streamt2=data01=$data01 + goto loop1 +endi + +if $data02 != 11 then + print ======streamt2=data02=$data02 + goto loop1 +endi + +system sh/stop_dnodes.sh \ No newline at end of file From 2e640c38a2df2114bb1733c8feeae0d112cab94e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 31 Oct 2022 18:37:28 +0800 Subject: [PATCH 26/60] refactor: stream and tmq message --- include/common/tmsg.h | 3 +- include/common/tmsgdef.h | 52 +++++++------ include/common/tname.h | 10 +-- include/libs/stream/tstream.h | 5 +- source/client/src/clientTmq.c | 10 +-- source/common/src/tdatablock.c | 36 ++++++--- source/common/src/tname.c | 8 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 24 +++--- source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 - source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 15 ++-- source/dnode/mnode/impl/src/mndConsumer.c | 18 ++--- source/dnode/mnode/impl/src/mndScheduler.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 72 ------------------ source/dnode/mnode/impl/src/mndSubscribe.c | 14 ++-- source/dnode/mnode/impl/src/mndTopic.c | 13 ++-- source/dnode/qnode/src/qnode.c | 4 +- source/dnode/snode/src/snode.c | 38 +++++++++- source/dnode/vnode/src/inc/vnodeInt.h | 12 ++- source/dnode/vnode/src/tq/tq.c | 83 ++++++++++----------- source/dnode/vnode/src/vnd/vnodeSvr.c | 42 ++++++----- source/libs/parser/src/parTranslater.c | 6 +- source/libs/stream/src/streamDispatch.c | 14 +++- source/libs/stream/src/streamRecover.c | 2 + 23 files changed, 234 insertions(+), 250 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b445aed244..5719db5420 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -57,7 +57,8 @@ extern int32_t tMsgDict[]; #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) #define TMSG_INFO(TYPE) \ ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \ - (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \ + (TYPE) < TDMT_VND_TMQ_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_VND_STREAM_MAX_MSG || \ + (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \ ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ : 0 #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index be4bf0e4d2..9c47bc702a 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -138,17 +138,15 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_INDEX, "get-table-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_BATCH_META, "batch-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_CFG, "table-cfg", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "alter-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "drop-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_HB, "consumer-hb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_TOPIC, "drop-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_HB, "consumer-hb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) @@ -186,21 +184,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset) - TD_DEF_MSG_TYPE(TDMT_VND_ADD_CHECK_INFO, "vnode-add-check-info", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DELETE_CHECK_INFO, "vnode-delete-check-info", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_DISPATCH_WRITE, "vnode-stream-task-dispatch-write", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP1, "vnode-stream-recover1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP2, "vnode-stream-recover2", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) @@ -232,15 +215,30 @@ enum { TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP1, "vnode-stream-recover1", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP2, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_MON_MSG) TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL) diff --git a/include/common/tname.h b/include/common/tname.h index 8c594a2b5e..666a25303e 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -72,13 +72,13 @@ bool tNameTbNameEqual(SName* left, SName* right); typedef struct { // input - SArray* tags; // element is SSmlKv - const char* sTableName; // super table name - uint8_t sTableNameLen; // the length of super table name + SArray* tags; // element is SSmlKv + const char* stbFullName; // super table name + uint8_t stbFullNameLen; // the length of super table name // output - char* childTableName; // must have size of TSDB_TABLE_NAME_LEN; - uint64_t uid; // child table uid, may be useful + char* ctbShortName; // must have size of TSDB_TABLE_NAME_LEN; + uint64_t uid; // child table uid, may be useful } RandTableName; void buildChildTableName(RandTableName* rName); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 534d86b1f1..e2616567f5 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -441,8 +441,9 @@ typedef struct { } SStreamRetrieveRsp; typedef struct { - int64_t streamId; - int32_t taskId; + SMsgHead msgHead; + int64_t streamId; + int32_t taskId; } SStreamRecoverStep1Req, SStreamRecoverStep2Req; typedef struct { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c18e1e3e40..ba639476d8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -508,7 +508,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb; - pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; + pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; // send msg atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); @@ -750,7 +750,7 @@ void tmqSendHbReq(void* param, void* tmrId) { sendInfo->requestObjRefId = 0; sendInfo->param = NULL; sendInfo->fp = tmqHbCb; - sendInfo->msgType = TDMT_MND_MQ_HB; + sendInfo->msgType = TDMT_MND_TMQ_HB; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1038,7 +1038,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { sendInfo->requestObjRefId = 0; sendInfo->param = ¶m; sendInfo->fp = tmqSubscribeCb; - sendInfo->msgType = TDMT_MND_SUBSCRIBE; + sendInfo->msgType = TDMT_MND_TMQ_SUBSCRIBE; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1420,7 +1420,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqAskEpCb; - sendInfo->msgType = TDMT_MND_MQ_ASK_EP; + sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1573,7 +1573,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqPollCb; - sendInfo->msgType = TDMT_VND_CONSUME; + sendInfo->msgType = TDMT_VND_TMQ_CONSUME; int64_t transporterId = 0; /*printf("send poll\n");*/ diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0c38d43543..502c2a6a08 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2063,8 +2063,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NULL, NULL, false, offset, - k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NULL, NULL, false, + offset, k); } else if (pCol->type == pColInfoData->info.type) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, k); @@ -2137,10 +2137,26 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } -char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { - ASSERT(stbName[0] != 0); +char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { + ASSERT(stbFullName[0] != 0); SArray* tags = taosArrayInit(0, sizeof(void*)); + if (tags == NULL) { + return NULL; + } + SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); + if (pTag == NULL) { + taosArrayDestroy(tags); + return NULL; + } + + void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); + if (cname == NULL) { + taosArrayDestroy(tags); + taosMemoryFree(pTag); + return NULL; + } + pTag->key = "group_id"; pTag->keyLen = strlen(pTag->key); pTag->type = TSDB_DATA_TYPE_UBIGINT; @@ -2148,13 +2164,11 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { pTag->length = sizeof(uint64_t); taosArrayPush(tags, &pTag); - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); - RandTableName rname = { .tags = tags, - .sTableName = stbName, - .sTableNameLen = strlen(stbName), - .childTableName = cname, + .stbFullName = stbFullName, + .stbFullNameLen = strlen(stbFullName), + .ctbShortName = cname, }; buildChildTableName(&rname); @@ -2162,8 +2176,8 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { taosMemoryFree(pTag); taosArrayDestroy(tags); - ASSERT(rname.childTableName && rname.childTableName[0]); - return rname.childTableName; + ASSERT(rname.ctbShortName && rname.ctbShortName[0]); + return rname.ctbShortName; } void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 70682d6cd4..4d83b6e3d8 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -315,7 +315,7 @@ static int compareKv(const void* p1, const void* p2) { */ void buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; - taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); + taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); @@ -336,11 +336,11 @@ void buildChildTableName(RandTableName* rName) { tMD5Final(&context); char temp[8] = {0}; - rName->childTableName[0] = 't'; - rName->childTableName[1] = '_'; + rName->ctbShortName[0] = 't'; + rName->ctbShortName[1] = '_'; for (int i = 0; i < 16; i++) { sprintf(temp, "%02x", context.digest[i]); - strcat(rName->childTableName, temp); + strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); rName->uid = *(uint64_t*)(context.digest); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 12e678886d..2c4a8e9b70 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -141,15 +141,13 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -171,10 +169,10 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ADD_CHECK_INFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE_CHECK_INFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 1ce7f1a84b..7ecb6fb208 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -72,8 +72,6 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d682fa9cc5..e2812e0d31 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -392,12 +392,12 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ADD_CHECK_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE_CHECK_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -409,10 +409,9 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index fa3651c7d7..aee1a7a75c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -63,12 +63,12 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_HB, mndProcessMqHbReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_LOST, mndProcessConsumerLostMsg); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndCancelGetNextConsumer); @@ -207,7 +207,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { pLostMsg->consumerId = pConsumer->consumerId; SRpcMsg pRpcMsg = { - .msgType = TDMT_MND_MQ_CONSUMER_LOST, + .msgType = TDMT_MND_TMQ_CONSUMER_LOST, .pCont = pLostMsg, .contLen = sizeof(SMqConsumerLostMsg), }; @@ -256,7 +256,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { mInfo("mq rebalance will be triggered"); SRpcMsg rpcMsg = { - .msgType = TDMT_MND_MQ_DO_REBALANCE, + .msgType = TDMT_MND_TMQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg), }; @@ -292,7 +292,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { pRecoverMsg->consumerId = consumerId; SRpcMsg pRpcMsg = { - .msgType = TDMT_MND_MQ_CONSUMER_RECOVER, + .msgType = TDMT_MND_TMQ_CONSUMER_RECOVER, .pCont = pRecoverMsg, .contLen = sizeof(SMqConsumerRecoverMsg), }; @@ -331,7 +331,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { pRecoverMsg->consumerId = consumerId; SRpcMsg pRpcMsg = { - .msgType = TDMT_MND_MQ_CONSUMER_RECOVER, + .msgType = TDMT_MND_TMQ_CONSUMER_RECOVER, .pCont = pRecoverMsg, .contLen = sizeof(SMqConsumerRecoverMsg), }; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 5ee4925a96..7e7c6ee0b6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -487,6 +487,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { qDestroyQueryPlan(pPlan); return -1; } + pTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(taskOneLevel, pTask); // source diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ff989e670..9f433f3322 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -557,78 +557,6 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -#if 0 -static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { - SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; - int32_t len; - int32_t code; - tEncodeSize(tEncodeSMStreamTaskRecoverReq, pReq, len, code); - if (code != 0) { - return -1; - } - void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - if (buf == NULL) { - return -1; - } - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SEncoder encoder; - tEncoderInit(&encoder, abuf, len); - tEncodeSMStreamTaskRecoverReq(&encoder, pReq); - ((SMsgHead *)buf)->vgId = pTask->nodeId; - - STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + len; - action.msgType = TDMT_STREAM_TASK_RECOVER; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - return 0; -} - -int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { - if (pStream->isDistributed) { - int32_t lv = taosArrayGetSize(pStream->tasks); - for (int32_t i = 0; i < lv; i++) { - SArray *pTasks = taosArrayGetP(pStream->tasks, i); - int32_t sz = taosArrayGetSize(pTasks); - SStreamTask *pTask = taosArrayGetP(pTasks, 0); - if (pTask->taskLevel == TASK_LEVEL__AGG) { - ASSERT(sz == 1); - if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { - return -1; - } - } else { - continue; - } - } - } else { - int32_t lv = taosArrayGetSize(pStream->tasks); - for (int32_t i = 0; i < lv; i++) { - SArray *pTasks = taosArrayGetP(pStream->tasks, i); - int32_t sz = taosArrayGetSize(pTasks); - for (int32_t j = 0; j < sz; j++) { - SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (pTask->taskLevel != TASK_LEVEL__SOURCE) break; - ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); - if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { - return -1; - } - } - } - } - return 0; -} -#endif - int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a6661e73c9..74f2b1288e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -73,11 +73,11 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP, mndProcessDropCgroupReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_SUBSCRIBE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DELETE_SUB_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DO_REBALANCE, mndProcessRebalanceReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP, mndProcessDropCgroupReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextSubscribe); @@ -164,7 +164,7 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM action.epSet = mndGetVgroupEpset(pMnode, pVgObj); action.pCont = buf; action.contLen = tlen; - action.msgType = TDMT_VND_MQ_VG_CHANGE; + action.msgType = TDMT_VND_TMQ_SUBSCRIBE; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -920,7 +920,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) action.epSet = pVgEp->epSet; action.pCont = pReq; action.contLen = sizeof(SMqVDeleteReq); - action.msgType = TDMT_VND_MQ_VG_DELETE; + action.msgType = TDMT_VND_TMQ_DELETE_SUB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 324404ce1b..e1ca1d2708 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -53,11 +53,10 @@ int32_t mndInitTopic(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTopicActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ADD_CHECK_INFO_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DELETE_CHECK_INFO_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CREATE_TOPIC, mndProcessCreateTopicReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_TOPIC, mndProcessDropTopicReq); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_ADD_CHECKINFO_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DEL_CHECKINFO_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); @@ -506,7 +505,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = buf; action.contLen = sizeof(SMsgHead) + len; - action.msgType = TDMT_VND_ADD_CHECK_INFO; + action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); @@ -715,7 +714,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = buf; action.contLen = sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN; - action.msgType = TDMT_VND_DELETE_CHECK_INFO; + action.msgType = TDMT_VND_TMQ_DEL_CHECKINFO; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 8dbfc14841..7e6bd75b2d 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -90,12 +90,12 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) { code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts); break; case TDMT_SCH_CANCEL_TASK: - //code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts); + // code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts); break; case TDMT_SCH_DROP_TASK: code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg, ts); break; - case TDMT_VND_CONSUME: + case TDMT_VND_TMQ_CONSUME: // code = tqProcessConsumeReq(pQnode->pTq, pMsg); // break; case TDMT_SCH_QUERY_HEARTBEAT: diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 719e2d8fad..d34159d312 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -261,18 +261,52 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return 0; } +int32_t sndProcessTaskRecoverFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { + char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamRecoverFinishReq req; + + SDecoder decoder; + tDecoderInit(&decoder, msg, msgLen); + tDecodeSStreamRecoverFinishReq(&decoder, &req); + tDecoderClear(&decoder); + + // find task + SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, req.taskId); + if (pTask == NULL) { + return -1; + } + // do process request + if (streamProcessRecoverFinishReq(pTask, req.childId) < 0) { + return -1; + } + + return 0; +} + +int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { + // + return 0; +} + int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: return sndProcessTaskRunReq(pSnode, pMsg); case TDMT_STREAM_TASK_DISPATCH: return sndProcessTaskDispatchReq(pSnode, pMsg, true); - case TDMT_STREAM_RETRIEVE: - return sndProcessTaskRetrieveReq(pSnode, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return sndProcessTaskDispatchRsp(pSnode, pMsg); + case TDMT_STREAM_RETRIEVE: + return sndProcessTaskRetrieveReq(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return sndProcessTaskRetrieveRsp(pSnode, pMsg); + case TDMT_STREAM_RECOVER_FINISH: + return sndProcessTaskRecoverFinishReq(pSnode, pMsg); + case TDMT_STREAM_RECOVER_FINISH_RSP: + return sndProcessTaskRecoverFinishRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index a5dc4431ab..12b4ee4139 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -179,8 +179,8 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); // tq-mq int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessVgDeleteReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream @@ -190,11 +190,15 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); -int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); +// int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); +// int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2c01645389..22ce4b20f5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -709,7 +709,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return 0; } -int32_t tqProcessVgDeleteReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { +int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; taosWLockLatch(&pTq->pushLock); @@ -767,7 +767,7 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t m return 0; } -int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { +int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); // todo lock @@ -982,25 +982,33 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg // 3.go through recover steps to fill history if (pTask->fillHistory) { - streamSetParamForRecover(pTask); if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + streamSetParamForRecover(pTask); streamSourceRecoverPrepareStep1(pTask, version); SStreamRecoverStep1Req req; streamBuildSourceRecover1Req(pTask, &req); - - void* serialziedReq = (void*)&req; int32_t len = sizeof(SStreamRecoverStep1Req); + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } + + memcpy(serializedReq, &req, len); + SRpcMsg rpcMsg = { .contLen = len, - .pCont = serialziedReq, + .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_STEP1, }; - tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg); + if (tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg) < 0) { + /*ASSERT(0);*/ + } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { + streamSetParamForRecover(pTask); streamAggRecoverPrepare(pTask); } else if (pTask->taskLevel == TASK_LEVEL__SINK) { // do nothing @@ -1010,8 +1018,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg return 0; } -int32_t tqProcessTaskRecover1Req(STQ* pTq, char* msg, int32_t msgLen) { - int32_t code; +int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { + int32_t code; + char* msg = pMsg->pCont; + int32_t msgLen = pMsg->contLen; + SStreamRecoverStep1Req* pReq = (SStreamRecoverStep1Req*)msg; SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, pReq->taskId); if (pTask == NULL) { @@ -1035,16 +1046,24 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, char* msg, int32_t msgLen) { return -1; } + ASSERT(pReq->taskId == pTask->taskId); + // serialize msg - int32_t len = sizeof(SStreamRecoverStep2Req); - void* serializedReq = (void*)&req; + int32_t len = sizeof(SStreamRecoverStep1Req); + + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } + + memcpy(serializedReq, &req, len); // dispatch msg SRpcMsg rpcMsg = { .code = 0, .contLen = len, .msgType = TDMT_VND_STREAM_RECOVER_STEP2, - .pCont = (void*)serializedReq, + .pCont = serializedReq, }; tmsgPutToQueue(&pTq->pVnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -1087,15 +1106,15 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t m return 0; } -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, char* msg, int32_t msgLen) { - int32_t code; +int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - int32_t len; SStreamRecoverFinishReq req; SDecoder decoder; - tDecoderInit(&decoder, msg, sizeof(SStreamRecoverFinishReq)); + tDecoderInit(&decoder, msg, msgLen); tDecodeSStreamRecoverFinishReq(&decoder, &req); tDecoderClear(&decoder); @@ -1112,6 +1131,11 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } +int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg) { + // + return 0; +} + int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { bool failed = false; SDecoder* pCoder = &(SDecoder){0}; @@ -1306,33 +1330,6 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { } } -#if 0 -int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { - SStreamTaskRecoverReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRecoverReq(pTask, pReq, pMsg); - return 0; - } else { - return -1; - } -} - -int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { - SStreamTaskRecoverRsp* pRsp = pMsg->pCont; - int32_t taskId = pRsp->rspTaskId; - - SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRecoverRsp(pTask, pRsp); - return 0; - } else { - return -1; - } -} -#endif - int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = pRsp->taskId; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4579f0c7a4..9c827c9fa3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -228,30 +228,30 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; /* TQ */ - case TDMT_VND_MQ_VG_CHANGE: - if (tqProcessVgChangeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + case TDMT_VND_TMQ_SUBSCRIBE: + if (tqProcessSubscribeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; - case TDMT_VND_MQ_VG_DELETE: - if (tqProcessVgDeleteReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + case TDMT_VND_TMQ_DELETE_SUB: + if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } break; - case TDMT_VND_MQ_COMMIT_OFFSET: + case TDMT_VND_TMQ_COMMIT_OFFSET: if (tqProcessOffsetCommitReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; - case TDMT_VND_ADD_CHECK_INFO: + case TDMT_VND_TMQ_ADD_CHECKINFO: if (tqProcessAddCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; - case TDMT_VND_DELETE_CHECK_INFO: + case TDMT_VND_TMQ_DEL_CHECKINFO: if (tqProcessDelCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; @@ -268,6 +268,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } } break; + case TDMT_VND_STREAM_RECOVER_STEP2: { + if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + goto _err; + } + } break; case TDMT_VND_ALTER_CONFIRM: vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; @@ -355,14 +360,11 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return 0; } - if (pMsg->msgType == TDMT_VND_CONSUME && !pVnode->restored) { + if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; } - char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - switch (pMsg->msgType) { case TDMT_SCH_FETCH: case TDMT_SCH_MERGE_FETCH: @@ -381,7 +383,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetTableCfg(pVnode, pMsg, true); case TDMT_VND_BATCH_META: return vnodeGetBatchMeta(pVnode, pMsg); - case TDMT_VND_CONSUME: + case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); @@ -389,16 +391,18 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); #endif - /*case TDMT_STREAM_TASK_RECOVER:*/ - /*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/ - case TDMT_STREAM_RETRIEVE: - return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); - /*case TDMT_STREAM_TASK_RECOVER_RSP:*/ - /*return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);*/ + case TDMT_STREAM_RETRIEVE: + return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_RECOVER_STEP1: + return tqProcessTaskRecover1Req(pVnode->pTq, pMsg); + case TDMT_STREAM_RECOVER_FINISH: + return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); + case TDMT_STREAM_RECOVER_FINISH_RSP: + return tqProcessTaskRecoverFinishRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bffeb06f7c..3f9fe264c2 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5279,7 +5279,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p code = buildCreateTopicReq(pCxt, pStmt, &createReq); } if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); + code = buildCmdMsg(pCxt, TDMT_MND_TMQ_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); } tFreeSCMCreateTopicReq(&createReq); return code; @@ -5293,7 +5293,7 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt tNameGetFullDbName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; - return buildCmdMsg(pCxt, TDMT_MND_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); + return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); } static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pStmt) { @@ -5305,7 +5305,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt dropReq.igNotExists = pStmt->ignoreNotExists; strcpy(dropReq.cgroup, pStmt->cgroup); - return buildCmdMsg(pCxt, TDMT_MND_MQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); + return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); } static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 417ddfa80d..7cccda51a1 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -239,7 +239,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; - msg.msgType = TDMT_VND_STREAM_RECOVER_FINISH; + msg.msgType = TDMT_STREAM_RECOVER_FINISH; tmsgSendReq(pEpSet, &msg); @@ -292,13 +292,19 @@ FAIL: int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, int64_t groupId) { - char* ctbName; + char* ctbName = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + if (ctbName == NULL) { + return -1; + } + if (pDataBlock->info.parTbName[0]) { - ctbName = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); } else { - ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); + char* ctbShortName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, ctbShortName); + taosMemoryFree(ctbShortName); } + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 12de2fafc1..c979ba058b 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -36,6 +36,7 @@ int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { } int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { + pReq->msgHead.vgId = pTask->nodeId; pReq->streamId = pTask->streamId; pReq->taskId = pTask->taskId; return 0; @@ -48,6 +49,7 @@ int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { } int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { + pReq->msgHead.vgId = pTask->nodeId; pReq->streamId = pTask->streamId; pReq->taskId = pTask->taskId; return 0; From 06c06d290f8878b30a34e9061d82dcbdf3ee3918 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 31 Oct 2022 18:45:10 +0800 Subject: [PATCH 27/60] opt http module --- source/libs/transport/inc/transComm.h | 1 + source/libs/transport/src/thttp.c | 280 +++++++++++++++++--------- source/libs/transport/src/trans.c | 2 + 3 files changed, 183 insertions(+), 100 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index b83f84e3f2..2354f0f959 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -428,6 +428,7 @@ void transDestoryExHandle(void* handle); int32_t transGetRefMgt(); int32_t transGetInstMgt(); +void transHttpEnvDestroy(); #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index fc61fb5122..9526ad472a 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -39,73 +39,11 @@ typedef struct SHttpMsg { char* cont; int32_t len; EHttpCompFlag flag; + int8_t quit; + SHttpModule* http; } SHttpMsg; -static TdThreadOnce transHttpInit = PTHREAD_ONCE_INIT; -static SHttpModule* http = NULL; - -static void httpHandleReq(SHttpMsg* msg); -static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, - EHttpCompFlag flag); - -static void* httpThread(void* arg) { - SHttpModule* http = (SHttpModule*)arg; - setThreadName("http-cli-send-thread"); - uv_run(http->loop, UV_RUN_DEFAULT); - return NULL; -} - -static void httpDestroyMsg(SHttpMsg* msg) { - if (msg == NULL) return; - - taosMemoryFree(msg->server); - taosMemoryFree(msg->cont); - taosMemoryFree(msg); -} -static void httpAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SHttpModule* http = item->pThrd; - - SHttpMsg* msg = NULL; - - queue wq; - taosThreadMutexLock(&item->mtx); - QUEUE_MOVE(&item->qmsg, &wq); - taosThreadMutexUnlock(&item->mtx); - - int count = 0; - while (!QUEUE_IS_EMPTY(&wq)) { - queue* h = QUEUE_HEAD(&wq); - QUEUE_REMOVE(h); - msg = QUEUE_DATA(h, SHttpMsg, q); - httpHandleReq(msg); - } -} - -static void transHttpEnvInit() { - http = taosMemoryMalloc(sizeof(SHttpModule)); - - http->loop = taosMemoryMalloc(sizeof(uv_loop_t)); - uv_loop_init(http->loop); - - http->asyncPool = transAsyncPoolCreate(http->loop, 1, http, httpAsyncCb); - - int err = taosThreadCreate(&http->thread, NULL, httpThread, (void*)http); - if (err != 0) { - taosMemoryFree(http->loop); - taosMemoryFree(http); - http = NULL; - } -} -static void transHttpEnvDestroy() { - if (http == NULL) return; - - transAsyncPoolDestroy(http->asyncPool); - taosMemoryFree(http->loop); - taosMemoryFree(http); -} - typedef struct SHttpClient { uv_connect_t conn; uv_tcp_t tcp; @@ -117,6 +55,17 @@ typedef struct SHttpClient { struct sockaddr_in dest; } SHttpClient; +static TdThreadOnce transHttpInit = PTHREAD_ONCE_INIT; +static SHttpModule* thttp = NULL; +static void transHttpEnvInit(); + +static void httpHandleReq(SHttpMsg* msg); +static void httpHandleQuit(SHttpMsg* msg); +static int32_t httpSendQuit(); + +static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag); + static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { if (flag == HTTP_FLAT) { @@ -135,6 +84,7 @@ static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pH "Content-Length: %d\n\n", server, contLen); } else { + terrno = TSDB_CODE_INVALID_CFG; return -1; } } @@ -208,6 +158,57 @@ _OVER: return code; } +static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { + uint32_t ip = taosGetIpv4FromFqdn(server); + if (ip == 0xffffffff) { + tError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr()); + return -1; + } + char buf[128] = {0}; + tinet_ntoa(buf, ip); + uv_ip4_addr(buf, port, dest); + return 0; +} + +static void* httpThread(void* arg) { + SHttpModule* http = (SHttpModule*)arg; + setThreadName("http-cli-send-thread"); + uv_run(http->loop, UV_RUN_DEFAULT); + return NULL; +} + +static void httpDestroyMsg(SHttpMsg* msg) { + if (msg == NULL) return; + + taosMemoryFree(msg->server); + taosMemoryFree(msg->cont); + taosMemoryFree(msg); +} +static void httpAsyncCb(uv_async_t* handle) { + SAsyncItem* item = handle->data; + SHttpModule* http = item->pThrd; + + SHttpMsg *msg = NULL, *quitMsg = NULL; + + queue wq; + taosThreadMutexLock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + taosThreadMutexUnlock(&item->mtx); + + int count = 0; + while (!QUEUE_IS_EMPTY(&wq)) { + queue* h = QUEUE_HEAD(&wq); + QUEUE_REMOVE(h); + msg = QUEUE_DATA(h, SHttpMsg, q); + if (msg->quit) { + quitMsg = msg; + } else { + httpHandleReq(msg); + } + } + if (quitMsg) httpHandleQuit(quitMsg); +} + static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { taosMemoryFree(cli->wbuf[0].base); taosMemoryFree(cli->wbuf[1].base); @@ -216,88 +217,82 @@ static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { taosMemoryFree(cli->addr); taosMemoryFree(cli); } + static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) { SHttpClient* cli = handle->data; destroyHttpClient(cli); } + static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SHttpClient* cli = handle->data; buf->base = cli->rbuf; buf->len = HTTP_RECV_BUF_SIZE; } + static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { SHttpClient* cli = handle->data; if (nread < 0) { - uError("http-report recv error:%s", uv_err_name(nread)); + tError("http-report recv error:%s", uv_err_name(nread)); } else { - uTrace("http-report succ to recv %d bytes", (int32_t)nread); + tTrace("http-report succ to recv %d bytes", (int32_t)nread); } if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); } } static void clientSentCb(uv_write_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { - uError("http-report failed to send data, reason: %s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + tError("http-report failed to send data, reason: %s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); } return; } else { - uTrace("http-report succ to send data"); + tTrace("http-report succ to send data"); } status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); if (status != 0) { - uError("http-report failed to recv data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + tError("http-report failed to recv data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); } } } static void clientConnCb(uv_connect_t* req, int32_t status) { SHttpClient* cli = req->data; if (status != 0) { - uError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + tError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); } return; } status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); if (0 != status) { - uError("http-report failed to send data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + tError("http-report failed to send data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); } } } -static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { - uint32_t ip = taosGetIpv4FromFqdn(server); - if (ip == 0xffffffff) { - uError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr()); +int32_t httpSendQuit() { + SHttpMsg* msg = taosMemoryCalloc(1, sizeof(SHttpMsg)); + msg->quit = 1; + + SHttpModule* load = atomic_load_ptr(&thttp); + if (load == NULL) { + httpDestroyMsg(msg); + tError("http-report already released"); return -1; + } else { + msg->http = load; } - char buf[128] = {0}; - tinet_ntoa(buf, ip); - uv_ip4_addr(buf, port, dest); + transAsyncSend(load->asyncPool, &(msg->q)); return 0; } -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { - taosThreadOnce(&transHttpInit, transHttpEnvInit); - return taosSendHttpReportImpl(server, port, pCont, contLen, flag); -} + static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); @@ -307,15 +302,49 @@ static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* p memcpy(msg->cont, pCont, contLen); msg->len = contLen; msg->flag = flag; + msg->quit = 0; + + SHttpModule* load = atomic_load_ptr(&thttp); + if (load == NULL) { + httpDestroyMsg(msg); + tError("http-report already released"); + return -1; + } else { + msg->http = load; + transAsyncSend(load->asyncPool, &(msg->q)); + } - transAsyncSend(http->asyncPool, &(msg->q)); return 0; } + +static void httpDestroyClientCb(uv_handle_t* handle) { + SHttpClient* http = handle->data; + destroyHttpClient(http); +} +static void httpWalkCb(uv_handle_t* handle, void* arg) { + // impl later + if (!uv_is_closing(handle)) { + uv_handle_type type = uv_handle_get_type(handle); + if (uv_handle_get_type(handle) == UV_TCP) { + uv_close(handle, httpDestroyClientCb); + } else { + uv_close(handle, NULL); + } + } + return; +} +static void httpHandleQuit(SHttpMsg* msg) { + SHttpModule* http = msg->http; + taosMemoryFree(msg); + + uv_walk(http->loop, httpWalkCb, NULL); +} static void httpHandleReq(SHttpMsg* msg) { + SHttpModule* http = msg->http; + struct sockaddr_in dest = {0}; - if (taosBuildDstAddr(msg->server, msg->port, &dest) < 0) { - httpDestroyMsg(msg); - return; + if (taosBuildDstAddr(msg->server, msg->port + 1, &dest) < 0) { + goto END; } if (msg->flag == HTTP_GZIP) { int32_t dstLen = taosCompressHttpRport(msg->cont, msg->len); @@ -324,11 +353,18 @@ static void httpHandleReq(SHttpMsg* msg) { } else { msg->flag = HTTP_FLAT; } + if (dstLen < 0) { + goto END; + } } int32_t len = 2048; char* header = taosMemoryCalloc(1, len); int32_t headLen = taosBuildHttpHeader(msg->server, msg->len, header, len, msg->flag); + if (headLen < 0) { + taosMemoryFree(header); + goto END; + } uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); wb[0] = uv_buf_init((char*)header, strlen(header)); // heap var @@ -347,20 +383,64 @@ static void httpHandleReq(SHttpMsg* msg) { taosMemoryFree(msg); uv_tcp_init(http->loop, &cli->tcp); + // set up timeout to avoid stuck; int32_t fd = taosCreateSocketWithTimeout(5); - - int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { - uError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); + tError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); destroyHttpClient(cli); return; } ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&cli->dest, clientConnCb); if (ret != 0) { - uError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, + tError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); destroyHttpClient(cli); } + return; + +END: + tError("http-report failed to report, reason: %s, addr: %s:%d", terrstr(), msg->server, msg->port); + httpDestroyMsg(msg); +} + +int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { + taosThreadOnce(&transHttpInit, transHttpEnvInit); + return taosSendHttpReportImpl(server, port, pCont, contLen, flag); +} + +static void transHttpEnvInit() { + SHttpModule* http = taosMemoryMalloc(sizeof(SHttpModule)); + + http->loop = taosMemoryMalloc(sizeof(uv_loop_t)); + uv_loop_init(http->loop); + + http->asyncPool = transAsyncPoolCreate(http->loop, 1, http, httpAsyncCb); + + int err = taosThreadCreate(&http->thread, NULL, httpThread, (void*)http); + if (err != 0) { + taosMemoryFree(http->loop); + taosMemoryFree(http); + http = NULL; + } + atomic_store_ptr(&thttp, http); +} + +void transHttpEnvDestroy() { + SHttpModule* load = atomic_load_ptr(&thttp); + if (load == NULL) { + return; + } + httpSendQuit(); + taosThreadJoin(load->thread, NULL); + + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg); + transAsyncPoolDestroy(load->asyncPool); + uv_loop_close(load->loop); + taosMemoryFree(load->loop); + taosMemoryFree(load); + + atomic_store_ptr(&thttp, NULL); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 756a8ff2cf..d3db4879d1 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -172,6 +172,8 @@ int32_t rpcInit() { } void rpcCleanup(void) { transCleanup(); + transHttpEnvDestroy(); + return; } From d21d88becc44ec8789b8acda81fb6af1ff3c5c60 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 31 Oct 2022 18:50:19 +0800 Subject: [PATCH 28/60] opt http module --- source/libs/transport/src/thttp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 9526ad472a..00854b5ee5 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -343,7 +343,7 @@ static void httpHandleReq(SHttpMsg* msg) { SHttpModule* http = msg->http; struct sockaddr_in dest = {0}; - if (taosBuildDstAddr(msg->server, msg->port + 1, &dest) < 0) { + if (taosBuildDstAddr(msg->server, msg->port, &dest) < 0) { goto END; } if (msg->flag == HTTP_GZIP) { From 1fd3ad502a1551163f83eb473514cead97d64f4b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 19:01:28 +0800 Subject: [PATCH 29/60] fix(query): ensure the capacity before loading tags value. --- source/libs/executor/src/scanoperator.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d499091ccb..750b9f258b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -382,6 +382,17 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo } } +static void ensureBlockCapacity(SSDataBlock* pBlock, int32_t capacity) { + // keep the value of rows temporarily + int32_t rows = pBlock->info.rows; + + pBlock->info.rows = 0; + blockDataEnsureCapacity(pBlock, capacity); + + // restore the rows number + pBlock->info.rows = rows; +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -413,6 +424,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + // NOTE: here the tag value only load for one row + ensureBlockCapacity(pBlock, 1); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; @@ -423,6 +436,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + ensureBlockCapacity(pBlock, 1); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); return TSDB_CODE_SUCCESS; } else { @@ -472,12 +486,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return terrno; } - int32_t rows = pBlock->info.rows; - pBlock->info.rows = 0; - - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - + ensureBlockCapacity(pBlock, pBlock->info.rows); relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); From ad857047aba317c8f375ae134dd7b463ef29a01b Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 31 Oct 2022 19:03:49 +0800 Subject: [PATCH 30/60] test:add replica 3 for all testcases and frameworks --- tests/system-test/0-others/udf_cluster.py | 2 +- tests/system-test/2-query/arccos.py | 2 +- tests/system-test/2-query/arcsin.py | 2 +- tests/system-test/2-query/arctan.py | 2 +- tests/system-test/2-query/cos.py | 2 +- tests/system-test/2-query/pow.py | 2 +- tests/system-test/2-query/queryQnode.py | 2 +- tests/system-test/2-query/sin.py | 2 +- tests/system-test/2-query/smaTest.py | 4 ++-- tests/system-test/2-query/sqrt.py | 2 +- tests/system-test/2-query/tan.py | 2 +- tests/system-test/6-cluster/5dnode1mnode.py | 2 +- tests/system-test/6-cluster/5dnode2mnode.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeDrop.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeDropInsert.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py | 2 +- .../6-cluster/5dnode3mnodeRestartDnodeInsertData.py | 2 +- .../6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py | 2 +- .../6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py | 2 +- .../6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py | 2 +- .../6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py | 2 +- .../6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 4 ++-- .../5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py | 6 +++--- .../6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py | 2 +- .../6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py | 2 +- .../6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py | 2 +- .../6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeStop.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeStopConnect.py | 2 +- .../6-cluster/5dnode3mnodeStopFollowerLeader.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeStopInsert.py | 2 +- tests/system-test/6-cluster/5dnode3mnodeStopLoop.py | 2 +- .../vnode/4dnode1mnode_basic_createDb_replica1.py | 2 +- .../vnode/4dnode1mnode_basic_replica1_insertdatas.py | 2 +- .../4dnode1mnode_basic_replica1_insertdatas_querys.py | 2 +- .../vnode/4dnode1mnode_basic_replica3_insertdatas.py | 2 +- ...de_basic_replica3_insertdatas_force_stop_all_dnodes.py | 2 +- .../4dnode1mnode_basic_replica3_insertdatas_querys.py | 2 +- ..._replica3_insertdatas_querys_loop_restart_all_vnode.py | 2 +- ...c_replica3_insertdatas_querys_loop_restart_follower.py | 2 +- ...sic_replica3_insertdatas_querys_loop_restart_leader.py | 2 +- ...de1mnode_basic_replica3_insertdatas_stop_all_dnodes.py | 2 +- ...mnode_basic_replica3_insertdatas_stop_follower_sync.py | 2 +- ...ode_basic_replica3_insertdatas_stop_follower_unsync.py | 2 +- ...eplica3_insertdatas_stop_follower_unsync_force_stop.py | 2 +- ...4dnode1mnode_basic_replica3_insertdatas_stop_leader.py | 2 +- ..._basic_replica3_insertdatas_stop_leader_forece_stop.py | 2 +- ...node1mnode_basic_replica3_mnode3_insertdatas_querys.py | 2 +- ...dnode1mnode_basic_replica3_querydatas_stop_follower.py | 2 +- ..._basic_replica3_querydatas_stop_follower_force_stop.py | 2 +- .../4dnode1mnode_basic_replica3_querydatas_stop_leader.py | 2 +- ...de_basic_replica3_querydatas_stop_leader_force_stop.py | 2 +- .../vnode/4dnode1mnode_basic_replica3_vgroups.py | 2 +- .../vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py | 2 +- tests/system-test/7-tmq/subscribeDb.py | 8 ++++---- 57 files changed, 64 insertions(+), 64 deletions(-) diff --git a/tests/system-test/0-others/udf_cluster.py b/tests/system-test/0-others/udf_cluster.py index 1ca1738332..a20bc128a2 100644 --- a/tests/system-test/0-others/udf_cluster.py +++ b/tests/system-test/0-others/udf_cluster.py @@ -19,7 +19,7 @@ class MyDnodes(TDDnodes): class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None self.depoly_cluster(3) diff --git a/tests/system-test/2-query/arccos.py b/tests/system-test/2-query/arccos.py index 1787521517..69e42a3911 100644 --- a/tests/system-test/2-query/arccos.py +++ b/tests/system-test/2-query/arccos.py @@ -12,7 +12,7 @@ class TDTestCase: # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/arcsin.py b/tests/system-test/2-query/arcsin.py index 127419029b..57d08d0587 100644 --- a/tests/system-test/2-query/arcsin.py +++ b/tests/system-test/2-query/arcsin.py @@ -12,7 +12,7 @@ class TDTestCase: # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/arctan.py b/tests/system-test/2-query/arctan.py index e6ae16b8d9..64a29f7ccb 100644 --- a/tests/system-test/2-query/arctan.py +++ b/tests/system-test/2-query/arctan.py @@ -12,7 +12,7 @@ class TDTestCase: # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index ab6814727e..2675c34266 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -12,7 +12,7 @@ class TDTestCase: # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py index 3a795ed25d..f803da6176 100644 --- a/tests/system-test/2-query/pow.py +++ b/tests/system-test/2-query/pow.py @@ -10,7 +10,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index 7bd573d7fc..e3f4732c08 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -64,7 +64,7 @@ class TDTestCase: return buildPath # init - def init(self, conn, logSql=True): + def init(self, conn, logSql=True, replicaVar=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) # tdSql.prepare() diff --git a/tests/system-test/2-query/sin.py b/tests/system-test/2-query/sin.py index a1ba335487..46b1ac50ae 100644 --- a/tests/system-test/2-query/sin.py +++ b/tests/system-test/2-query/sin.py @@ -10,7 +10,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/smaTest.py b/tests/system-test/2-query/smaTest.py index 66a89d8701..20ed103f8e 100644 --- a/tests/system-test/2-query/smaTest.py +++ b/tests/system-test/2-query/smaTest.py @@ -78,7 +78,7 @@ class TDTestCase: def insert_data(self, tbname, ts_start, count): pre_insert = "insert into %s values"%tbname sql = pre_insert - tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + tdLog.debug("insert table %s rows=%d ..."%(tbname, count)) for i in range(count): sql += " (%d,%d)"%(ts_start + i*1000, i ) if i >0 and i%30000 == 0: @@ -94,7 +94,7 @@ class TDTestCase: def insert_data1(self, tbname, ts_start, count): pre_insert = "insert into %s values"%tbname sql = pre_insert - tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + tdLog.debug("insert table %s rows=%d ..."%(tbname, count)) for i in range(count): sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) if i >0 and i%30000 == 0: diff --git a/tests/system-test/2-query/sqrt.py b/tests/system-test/2-query/sqrt.py index 9597375885..4784b71fc6 100644 --- a/tests/system-test/2-query/sqrt.py +++ b/tests/system-test/2-query/sqrt.py @@ -10,7 +10,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/tan.py b/tests/system-test/2-query/tan.py index 683cee37ff..203f149712 100644 --- a/tests/system-test/2-query/tan.py +++ b/tests/system-test/2-query/tan.py @@ -10,7 +10,7 @@ from util.cases import * class TDTestCase: - def init(self, conn, powSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 367d0bfbd7..f2c0bf9902 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -21,7 +21,7 @@ class MyDnodes(TDDnodes): class TDTestCase: noConn = True - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None self.depoly_cluster(5) diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index 7cbadde8bf..6054ef69f8 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -20,7 +20,7 @@ from clusterCommonCreate import * from clusterCommonCheck import * class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py index 909decc195..e591114ef4 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py +++ b/tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py @@ -26,7 +26,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeDrop.py b/tests/system-test/6-cluster/5dnode3mnodeDrop.py index 33dc3ebaac..52f734b534 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDrop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDrop.py @@ -22,7 +22,7 @@ class MyDnodes(TDDnodes): class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None diff --git a/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py b/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py index 74a7c32206..2b013704f2 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeDropInsert.py @@ -26,7 +26,7 @@ class MyDnodes(TDDnodes): class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None self.ts = 1500000000000 diff --git a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py index 2e107561b5..1590a5948b 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py @@ -26,7 +26,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 3fe6a689cb..52c9773544 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -26,7 +26,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index 0d56729280..4d641b0e27 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -26,7 +26,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index a215a74559..eba456d1c2 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py index 9df1ffb3df..1949157191 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py index caaf1dd898..382144b69c 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py @@ -26,7 +26,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py index 575e2ef712..9acb313203 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql,replicaVar=1): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) @@ -171,7 +171,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='mnode') + self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=1,stopRole='mnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py index d24c749515..e8e3b217a1 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py @@ -25,12 +25,12 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql, replicaVar=1): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) self.host = socket.gethostname() - + self.replicaVar = replicaVar def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -88,7 +88,7 @@ class TDTestCase: vnodeNumbers = int(dnodeNumbers-mnodeNums) allDbNumbers=(paraDict['dbNumbers']*restartNumbers) allStbNumbers=(paraDict['stbNumbers']*restartNumbers) - + paraDict['replica'] = self.replicaVar tdLog.info("first check dnode and mnode") tdSql.query("select * from information_schema.ins_dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py index 9a94632a12..14ae1a8a48 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py index 6fbe262b8e..056e3225e8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py index 56bafe8abc..5fd79e4296 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None tdSql.init(conn.cursor()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py index 539e5e38a7..a34895ff42 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py @@ -25,7 +25,7 @@ import ctypes class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") # tdSql.init(conn.cursor()) # self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index c43d41199b..531688710a 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -24,7 +24,7 @@ from multiprocessing import Process class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py index 584dd645f6..76ff746b2e 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop2Follower.py @@ -24,7 +24,7 @@ from multiprocessing import Process class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py b/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py index 5638b01070..142f88c0d9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopConnect.py @@ -25,7 +25,7 @@ from multiprocessing import Process class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py b/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py index 6daa4c260a..afa1c3dcc3 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopFollowerLeader.py @@ -24,7 +24,7 @@ from multiprocessing import Process class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py index dcd9e56a64..780255604d 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopInsert.py @@ -26,7 +26,7 @@ class MyDnodes(TDDnodes): class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py index 1644686568..97134ac2d1 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py @@ -24,7 +24,7 @@ from multiprocessing import Process class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py index 8d07dc6b38..435701bd8f 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -18,7 +18,7 @@ import subprocess sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py index 470278cd8e..f0d5e8328a 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -18,7 +18,7 @@ import subprocess sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py index 0484ee9f33..8d0801500e 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py @@ -18,7 +18,7 @@ import subprocess ,threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py index 60e3167304..379615d358 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py @@ -18,7 +18,7 @@ import subprocess sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py index 0557836ab2..240169dbb6 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_force_stop_all_dnodes.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py index 8523ac72a4..25b5de5afa 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py @@ -18,7 +18,7 @@ import subprocess ,threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py index 8e7ac388fb..ab5359601a 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_all_vnode.py @@ -18,7 +18,7 @@ import subprocess ,threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py index 37c00c4b8d..4eec3c348d 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_follower.py @@ -18,7 +18,7 @@ import subprocess ,threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py index f1ff805f08..0fe18ab705 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys_loop_restart_leader.py @@ -18,7 +18,7 @@ import subprocess ,threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py index 07231555fe..75a7ca51b6 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_all_dnodes.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py index 80d367db01..e9c63151f3 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py index 55a2318817..5dd57542b3 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py index 489e2acd43..fd00e89216 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index ee0ab26f4c..f98e2c07b5 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -19,7 +19,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py index c7895abe04..5be2b67c31 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py @@ -19,7 +19,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py index aca188824d..e8fa3099e9 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_mnode3_insertdatas_querys.py @@ -18,7 +18,7 @@ import subprocess ,threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py index fa69643079..21d0c2d2b8 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py index 0ef8db9c0f..ff6ce98c81 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_follower_force_stop.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py index f26df70c4e..e36f488c2b 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py index 2f57af39b0..050b98024d 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_querydatas_stop_leader_force_stop.py @@ -21,7 +21,7 @@ import threading sys.path.append(os.path.dirname(__file__)) class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py index 221053d165..d086743306 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -17,7 +17,7 @@ import socket import subprocess class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py index 05ec8efcd2..b9d3afa765 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -19,7 +19,7 @@ import socket import subprocess class TDTestCase: - def init(self,conn ,logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) self.host = socket.gethostname() diff --git a/tests/system-test/7-tmq/subscribeDb.py b/tests/system-test/7-tmq/subscribeDb.py index f738f816c7..3790bd1690 100644 --- a/tests/system-test/7-tmq/subscribeDb.py +++ b/tests/system-test/7-tmq/subscribeDb.py @@ -209,7 +209,7 @@ class TDTestCase: # wait for data ready prepareEnvThread.join() - tdLog.info("insert process end, and start to check consume result") + tdLog.info("1-insert process end, and start to check consume result") expectRows = 1 resultList = self.selectConsumeResult(expectRows) totalConsumeRows = 0 @@ -310,7 +310,7 @@ class TDTestCase: # wait for data ready prepareEnvThread.join() - tdLog.info("insert process end, and start to check consume result") + tdLog.info("2-insert process end, and start to check consume result") expectRows = 2 resultList = self.selectConsumeResult(expectRows) totalConsumeRows = 0 @@ -379,7 +379,7 @@ class TDTestCase: # wait for data ready prepareEnvThread.join() - tdLog.info("insert process end, and start to check consume result") + tdLog.info("3-insert process end, and start to check consume result") expectRows = 2 resultList = self.selectConsumeResult(expectRows) totalConsumeRows = 0 @@ -459,7 +459,7 @@ class TDTestCase: prepareEnvThread.join() prepareEnvThread2.join() - tdLog.info("insert process end, and start to check consume result") + tdLog.info("4-insert process end, and start to check consume result") expectRows = 1 resultList = self.selectConsumeResult(expectRows) totalConsumeRows = 0 From ece67e2e0cf071b5bc9a1c759e7960578fbfea8d Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 31 Oct 2022 19:09:08 +0800 Subject: [PATCH 31/60] test:add replica 3 for all testcases and frameworks --- .../taosbenchmark/auto_create_table_json.py | 2 +- .../5-taos-tools/taosbenchmark/commandline.py | 2 +- .../taosbenchmark/custom_col_tag.py | 2 +- .../taosbenchmark/default_json.py | 2 +- .../5-taos-tools/taosbenchmark/demo.py | 2 +- .../taosbenchmark/insert_alltypes_json.py | 2 +- .../taosbenchmark/invalid_commandline.py | 2 +- .../5-taos-tools/taosbenchmark/json_tag.py | 2 +- .../taosbenchmark/limit_offset_json.py | 2 +- .../5-taos-tools/taosbenchmark/query_json.py | 2 +- .../taosbenchmark/sample_csv_json.py | 2 +- .../taosbenchmark/sml_interlace.py | 2 +- .../taosbenchmark/sml_json_alltypes.py | 2 +- .../taosbenchmark/sml_telnet_alltypes.py | 2 +- .../taosbenchmark/taosadapter_json.py | 2 +- .../5-taos-tools/taosbenchmark/telnet_tcp.py | 2 +- .../taosdump/taosdumpTestInspect.py | 2 +- .../taosdump/taosdumpTestTypeBigInt.py | 2 +- .../taosdump/taosdumpTestTypeBinary.py | 2 +- .../taosdump/taosdumpTestTypeBool.py | 2 +- .../taosdump/taosdumpTestTypeDouble.py | 2 +- .../taosdump/taosdumpTestTypeFloat.py | 2 +- .../taosdump/taosdumpTestTypeInt.py | 2 +- .../taosdump/taosdumpTestTypeJson.py | 2 +- .../taosdump/taosdumpTestTypeSmallInt.py | 2 +- .../taosdump/taosdumpTestTypeTinyInt.py | 2 +- .../taosdumpTestTypeUnsignedBigInt.py | 2 +- .../taosdump/taosdumpTestTypeUnsignedInt.py | 2 +- .../taosdumpTestTypeUnsignedSmallInt.py | 2 +- .../taosdumpTestTypeUnsignedTinyInt.py | 2 +- tests/system-test/fulltest.sh | 61 ++++++++++--------- 31 files changed, 61 insertions(+), 60 deletions(-) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py index 8509136a34..23b6341e66 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py @@ -25,7 +25,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py index 1d21c517e6..4473ff2c2d 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py b/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py index 9104d63096..4f8e94a887 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index c68375894c..95afdc413e 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -23,7 +23,7 @@ class TDTestCase: [TD-11510] taosBenchmark test cases """ - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index a44ad4c1d0..90d0ffd8b1 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -27,7 +27,7 @@ class TDTestCase: """ return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py index 38332f7b64..7437b46353 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py index ebddff5643..a706967bc1 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py b/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py index 5f25e0229b..3b9123974f 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py index 47a60e4757..dbddb840ca 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/limit_offset_json.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py index 84d9433967..7f0b082e33 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py @@ -29,7 +29,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py index 772bb11df4..5b59d84b79 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sample_csv_json.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py index fec9376529..ee7bb8a925 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_interlace.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py index 557d2a4884..d7ccdc2716 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py index 8ff56b5339..7b0f5d1672 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_telnet_alltypes.py @@ -24,7 +24,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py index 3f0a05b665..1588b4077c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/taosadapter_json.py @@ -25,7 +25,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py b/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py index 7aa0575f77..b8d4c35e58 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/telnet_tcp.py @@ -25,7 +25,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py index 33ba4034ec..09976953e1 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py index 75885edf9f..7eaddd9e2d 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBigInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py index 41d4ba128d..c70748e9c2 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBinary.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py index 043fb49af3..eb4b8a364a 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeBool.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py index 1eaa2bbfff..70501965fe 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeDouble.py @@ -28,7 +28,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py index 4373aa5669..5e551a373c 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeFloat.py @@ -28,7 +28,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py index 45009a716a..f2d4c4f814 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py index d1c4313c40..5971902cb6 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeJson.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py index 941de25683..f2dca23488 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeSmallInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py index e5392d38fe..6f2781d645 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeTinyInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py index dd8dd5d76d..b226204654 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedBigInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py index eacd1bb7dd..766110d74e 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py index ea362ad2b4..242681b0fc 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedSmallInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py index a3ffc13d62..d512cba97c 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestTypeUnsignedTinyInt.py @@ -27,7 +27,7 @@ class TDTestCase: ''' return - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVarl=1): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 6c717f347c..8529f8d99b 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,35 +2,35 @@ set -e set -x -# python3 ./test.py -f 0-others/taosShell.py -# python3 ./test.py -f 0-others/taosShellError.py -# python3 ./test.py -f 0-others/taosShellNetChk.py -# python3 ./test.py -f 0-others/telemetry.py -# python3 ./test.py -f 0-others/taosdMonitor.py -# python3 ./test.py -f 0-others/udfTest.py -# python3 ./test.py -f 0-others/udf_create.py -# python3 ./test.py -f 0-others/udf_restart_taosd.py -# python3 ./test.py -f 0-others/cachemodel.py -# python3 ./test.py -f 0-others/udf_cfg1.py -# python3 ./test.py -f 0-others/udf_cfg2.py -# python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 -# python3 ./test.py -f 0-others/sysinfo.py -# python3 ./test.py -f 0-others/user_control.py -# python3 ./test.py -f 0-others/fsync.py -# python3 ./test.py -f 0-others/compatibility.py -# python3 ./test.py -f 1-insert/alter_database.py -# python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -# python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -# python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -# python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -# python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -# python3 ./test.py -f 1-insert/alter_stable.py -# python3 ./test.py -f 1-insert/alter_table.py -# python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -# python3 ./test.py -f 1-insert/table_comment.py -# python3 ./test.py -f 1-insert/time_range_wise.py -# python3 ./test.py -f 1-insert/block_wise.py -# python3 ./test.py -f 1-insert/create_retentions.py +python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShellError.py +python3 ./test.py -f 0-others/taosShellNetChk.py +python3 ./test.py -f 0-others/telemetry.py +python3 ./test.py -f 0-others/taosdMonitor.py +python3 ./test.py -f 0-others/udfTest.py +python3 ./test.py -f 0-others/udf_create.py +python3 ./test.py -f 0-others/udf_restart_taosd.py +python3 ./test.py -f 0-others/cachemodel.py +python3 ./test.py -f 0-others/udf_cfg1.py +python3 ./test.py -f 0-others/udf_cfg2.py +python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +python3 ./test.py -f 0-others/sysinfo.py +python3 ./test.py -f 0-others/user_control.py +python3 ./test.py -f 0-others/fsync.py +python3 ./test.py -f 0-others/compatibility.py +python3 ./test.py -f 1-insert/alter_database.py +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +python3 ./test.py -f 1-insert/alter_stable.py +python3 ./test.py -f 1-insert/alter_table.py +python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +python3 ./test.py -f 1-insert/table_comment.py +python3 ./test.py -f 1-insert/time_range_wise.py +python3 ./test.py -f 1-insert/block_wise.py +python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py python3 ./test.py -f 1-insert/mutil_stage.py python3 ./test.py -f 1-insert/table_param_ttl.py -R @@ -239,7 +239,8 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 6 -M 3 -n 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 From 8ba93cf9519b04f986867f1d61dcc72e871ee6ca Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 31 Oct 2022 19:18:41 +0800 Subject: [PATCH 32/60] test:add replica 3 for all testcases and frameworks --- tests/develop-test/test.py | 81 +++++++++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 18 deletions(-) diff --git a/tests/develop-test/test.py b/tests/develop-test/test.py index 5dc6139410..b25bda4a3b 100644 --- a/tests/develop-test/test.py +++ b/tests/develop-test/test.py @@ -72,8 +72,9 @@ if __name__ == "__main__": queryPolicy = 1 createDnodeNums = 1 restful = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate']) + replicaVar = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -95,8 +96,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-D taosadapter update cfg dict ') - - + tdLog.printNoPrefix('-n the number of replicas') sys.exit(0) if key in ['-r', '--restart']: @@ -168,6 +168,9 @@ if __name__ == "__main__": print('adapter cfg update convert fail.') sys.exit(0) + if key in ['-n', '--replicaVar']: + replicaVar = value + if not execCmd == "": if restful: tAdapter.init(deployPath) @@ -194,7 +197,7 @@ if __name__ == "__main__": processID = subprocess.check_output(psCmd, shell=True) for port in range(6030, 6041): - usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + usePortPID = "lsof -i tcp:%d | grep LISTEN | awk '{print $2}'" % port processID = subprocess.check_output(usePortPID, shell=True) if processID: @@ -206,11 +209,13 @@ if __name__ == "__main__": time.sleep(2) if restful: - toBeKilled = "taosadapter" + toBeKilled = "taosadapt" - killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + # killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + killCmd = f"pkill {toBeKilled}" psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + # psCmd = f"pgrep {toBeKilled}" processID = subprocess.check_output(psCmd, shell=True) while(processID): @@ -218,15 +223,15 @@ if __name__ == "__main__": time.sleep(1) processID = subprocess.check_output(psCmd, shell=True) - for port in range(6030, 6041): - usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port - processID = subprocess.check_output(usePortPID, shell=True) + port = 6041 + usePortPID = f"lsof -i tcp:{port} | grep LISTEN | awk '{{print $2}}'" + processID = subprocess.check_output(usePortPID, shell=True) - if processID: - killCmd = "kill -TERM %s" % processID - os.system(killCmd) - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) + if processID: + killCmd = f"kill -TERM {processID}" + os.system(killCmd) + fuserCmd = f"fuser -k -n tcp {port}" + os.system(fuserCmd) tdLog.info('stop taosadapter') @@ -319,7 +324,7 @@ if __name__ == "__main__": for dnode in tdDnodes.dnodes: tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) - + if restful: tAdapter.deploy(adapter_cfg_dict) tAdapter.start() @@ -339,6 +344,26 @@ if __name__ == "__main__": print("check dnode ready") except Exception as r: print(r) + if queryPolicy != 1: + queryPolicy=int(queryPolicy) + if restful: + conn = taosrest.connect(url=f"http://{host}:6041") + else: + conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + + cursor = conn.cursor() + cursor.execute("create qnode on dnode 1") + cursor.execute(f'alter local "queryPolicy" "{queryPolicy}"') + cursor.execute("show local variables") + res = cursor.fetchall() + for i in range(cursor.rowcount): + if res[i][0] == "queryPolicy" : + if int(res[i][1]) == int(queryPolicy): + tdLog.success(f'alter queryPolicy to {queryPolicy} successfully') + else: + tdLog.debug(res) + tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") + if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: conn = None else: @@ -453,6 +478,26 @@ if __name__ == "__main__": except Exception as r: print(r) + if queryPolicy != 1: + queryPolicy=int(queryPolicy) + if restful: + conn = taosrest.connect(url=f"http://{host}:6041") + else: + conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) + + cursor = conn.cursor() + cursor.execute("create qnode on dnode 1") + cursor.execute(f'alter local "queryPolicy" "{queryPolicy}"') + cursor.execute("show local variables") + res = cursor.fetchall() + for i in range(cursor.rowcount): + if res[i][0] == "queryPolicy" : + if int(res[i][1]) == int(queryPolicy): + tdLog.success(f'alter queryPolicy to {queryPolicy} successfully') + else: + tdLog.debug(res) + tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") + if testCluster: tdLog.info("Procedures for testing cluster") @@ -470,7 +515,7 @@ if __name__ == "__main__": if fileName == "all": tdCases.runAllLinux(conn) else: - tdCases.runOneLinux(conn, fileName) + tdCases.runOneLinux(conn, fileName, replicaVar) if restart: if fileName == "all": @@ -487,7 +532,7 @@ if __name__ == "__main__": conn = taosrest.connect(url=f"http://{host}:6041") tdLog.info("Procedures for tdengine deployed in %s" % (host)) tdLog.info("query test after taosd restart") - tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py", replicaVar) else: tdLog.info("not need to query") From 8a5f94d35c90a5c730fba40456a80c67f128435b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 31 Oct 2022 20:03:21 +0800 Subject: [PATCH 33/60] fix: set search key according to order --- source/dnode/vnode/src/tsdb/tsdbRead.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f90c567be..3d6b16d5c7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -821,7 +821,8 @@ int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SData } else if (!asc && pReader->window.skey <= pBlock->minKey.ts) { endPos = 0; } else { - endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.ekey, pReader->order); + int64_t key = asc ? pReader->window.ekey : pReader->window.skey; + endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, pReader->order); } return endPos; @@ -852,8 +853,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // pDumpInfo->rowIndex = pBlock->nRow - 1; } else { int32_t pos = asc ? pBlock->nRow - 1 : 0; - int32_t order = (pReader->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.skey, order); + int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int64_t key = asc ? pReader->window.skey : pReader->window.ekey; + pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, order); } } From c609493cbb40e9a561eba0b1db241d6e5bdd6679 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Mon, 31 Oct 2022 20:57:18 +0800 Subject: [PATCH 34/60] test:modify tmq test case for replica3 --- tests/system-test/7-tmq/tmqSubscribeStb-r3.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index c5f98bc3a0..f49464f5be 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -100,7 +100,7 @@ class TDTestCase: # tmqCom.insert_data_with_autoCreateTbl(tsql=tdSql,dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix="ctbx", # ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], # startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - tmqCom.asyncInsertDataByInterlace(paraDict) + pThread = tmqCom.asyncInsertDataByInterlace(paraDict) tdLog.info("wait some data inserted") exitFlag = 1 @@ -128,6 +128,7 @@ class TDTestCase: cluster.dnodes[4].stoptaosd() cluster.dnodes[4].starttaosd() + pThread.join() # tdLog.info("restart taosd to ensure that the data falls into the disk") # tdSql.query("flush database %s"%(paraDict['dbName'])) return @@ -290,6 +291,7 @@ class TDTestCase: def run(self): self.prepareTestEnv() self.tmqCase1() + self.prepareTestEnv() self.tmqCase2() def stop(self): From 1d0eae197dae26e0568f2307b2f0c70c6cf350ec Mon Sep 17 00:00:00 2001 From: "benguang.zhao" Date: Mon, 31 Oct 2022 21:06:08 +0800 Subject: [PATCH 35/60] fix: not allow to rollback WAL across snapshot point --- source/libs/wal/src/walWrite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 527ffa0056..e016e72471 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -101,7 +101,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosThreadMutexLock(&pWal->mutex); int64_t code; char fnameStr[WAL_FILE_LEN]; - if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { terrno = TSDB_CODE_WAL_INVALID_VER; taosThreadMutexUnlock(&pWal->mutex); return -1; From 96520291216455d3e5630b5de82e7b0e2e5d8167 Mon Sep 17 00:00:00 2001 From: "benguang.zhao" Date: Mon, 31 Oct 2022 21:08:35 +0800 Subject: [PATCH 36/60] fix: update testcases of walRollback in walMetaTest.cpp --- source/libs/wal/test/walMetaTest.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index d98826a384..891e7dcdae 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -274,17 +274,17 @@ TEST_F(WalCleanEnv, rollbackMultiFile) { code = walRollback(pWal, 9); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 6); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); code = walRollback(pWal, 5); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 4); - code = walRollback(pWal, 3); - ASSERT_EQ(code, 0); + ASSERT_EQ(code, -1); - ASSERT_EQ(pWal->vers.lastVer, 2); + ASSERT_EQ(pWal->vers.lastVer, 5); - code = walWrite(pWal, 3, 3, (void*)ranStr, ranStrLen); + code = walWrite(pWal, 6, 6, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 3); + ASSERT_EQ(pWal->vers.lastVer, 6); code = walSaveMeta(pWal); ASSERT_EQ(code, 0); From c17a921a03f94da375ce54b5eb144e7dd4b1683b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 31 Oct 2022 23:19:40 +0800 Subject: [PATCH 37/60] fix(query): set the correct buffer to accommodate the tag values. --- source/common/src/tdatablock.c | 1 + source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/cachescanoperator.c | 17 ++++----- source/libs/executor/src/scanoperator.c | 37 +++++++++++--------- source/libs/function/src/builtinsimpl.c | 2 +- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0c38d43543..3afa91f2f9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1186,6 +1186,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { + pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { pColumn->varmeta.length = 0; if (pColumn->varmeta.offset != NULL) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1191b6a485..16f094c85d 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -921,7 +921,7 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SColMatchInfo* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock, const char* idStr); + SSDataBlock* pBlock, int32_t rows, const char* idStr); void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 92e98d3eab..bcd59e83f0 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -152,25 +152,27 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->indexOfBufferedRes = 0; } + SSDataBlock* pRes = pInfo->pRes; + if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { SColMatchItem* pMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i); int32_t slotId = pMatchInfo->dstSlotId; SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId); - SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, slotId); + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId); char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); bool isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes); colDataAppend(pDst, 0, p, isNull); } - pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); - pInfo->pRes->info.rows = 1; + pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + pRes->info.rows = 1; if (pInfo->pseudoExprSup.numOfExprs > 0) { SExprSupp* pSup = &pInfo->pseudoExprSup; - int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, pRes->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; @@ -178,10 +180,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pRes->info.groupId = getTableGroupId(pTableList, pInfo->pRes->info.uid); - + pRes->info.groupId = getTableGroupId(pTableList, pRes->info.uid); pInfo->indexOfBufferedRes += 1; - return pInfo->pRes; + return pRes; } else { doSetOperatorCompleted(pOperator); return NULL; @@ -221,7 +222,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); - code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 750b9f258b..b3226f292a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -344,11 +344,11 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } -static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, int32_t rows) { if (pTableScanInfo->pseudoSup.numOfExprs > 0) { SExprSupp* pSup = &pTableScanInfo->pseudoSup; - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -426,7 +426,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca // NOTE: here the tag value only load for one row ensureBlockCapacity(pBlock, 1); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { @@ -437,7 +437,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); ensureBlockCapacity(pBlock, 1); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -488,7 +488,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca ensureBlockCapacity(pBlock, pBlock->info.rows); relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value pCost->totalRows -= pBlock->info.rows; @@ -528,27 +528,30 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction } int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock, const char* idStr) { + SSDataBlock* pBlock, int32_t rows, const char* idStr) { // currently only the tbname pseudo column if (numOfPseudoExpr == 0) { return TSDB_CODE_SUCCESS; } + // backup the rows + int32_t backupRows = pBlock->info.rows; + pBlock->info.rows = rows; + SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid); + metaReaderReleaseLock(&mr); + if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); metaReaderClear(&mr); return terrno; } - metaReaderReleaseLock(&mr); - for (int32_t j = 0; j < numOfPseudoExpr; ++j) { SExprInfo* pExpr = &pPseudoExpr[j]; - - int32_t dstSlotId = pExpr->base.resSchema.slotId; + int32_t dstSlotId = pExpr->base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); colInfoDataCleanup(pColInfoData, pBlock->info.rows); @@ -587,6 +590,9 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int } metaReaderClear(&mr); + + // restore the rows + pBlock->info.rows = backupRows; return TSDB_CODE_SUCCESS; } @@ -631,7 +637,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { blockDataCleanup(pBlock); - SDataBlockInfo* pBInfo = &pBlock->info; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBInfo->rows, &pBInfo->uid, &pBInfo->window); @@ -1167,7 +1172,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU pBlock->info.rows = rows; relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, rows); pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->uid); } @@ -1638,7 +1643,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, - GET_TASKID(pTaskInfo)); + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { blockDataFreeRes((SSDataBlock*)pBlock); T_LONG_JMP(pTaskInfo->env, code); @@ -1648,11 +1653,11 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock if (filter) { doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); } + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); calBlockTbName(&pInfo->tbnameCalSup, pInfo->pRes); - return 0; } @@ -4368,7 +4373,7 @@ static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeS // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo, - pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo)); + pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -4485,7 +4490,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo, - pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo)); + pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5dcef7cd17..b9acd36088 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -497,7 +497,7 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static FORCE_INLINE int32_t getNumOfElems(SqlFunctionCtx* pCtx) { +static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; /* From 4356fbc3c5888b126f69a4890c728265d00db31e Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 1 Nov 2022 08:58:52 +0800 Subject: [PATCH 38/60] fix: error of setting ld_library_path --- source/libs/function/src/tudf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index eff3c6cc7b..459ee583a4 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -137,7 +137,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { fnError("can not set correct udfd LD_LIBRARY_PATH"); } char ldLibPathEnvItem[1024 + 32] = {0}; - snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", ldLibPathEnvItem); + snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", udfdPathLdLib); char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem, NULL}; options.env = envUdfd; From be467a10c71ca7fe09cc4aed514930d2d8428413 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 1 Nov 2022 09:36:33 +0800 Subject: [PATCH 39/60] fix: drop index error --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d76ee55ef4..6aaa8cc98f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5160,7 +5160,7 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->indexName, &name), dropSmaReq.name); + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); dropSmaReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); } From 2e950c66968a72a24c8b8358095f4d44ec024a0c Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 10:06:09 +0800 Subject: [PATCH 40/60] chore: comment out tfs_test and dbTest --- source/dnode/mnode/impl/test/db/CMakeLists.txt | 9 ++++----- source/libs/tfs/test/CMakeLists.txt | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt index 69e36cae08..04bd0d984d 100644 --- a/source/dnode/mnode/impl/test/db/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -5,8 +5,7 @@ target_link_libraries( PUBLIC sut ) -# -#add_test( -# NAME dbTest -# COMMAND dbTest -#) +add_test( + NAME dbTest + COMMAND dbTest +) diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt index 977b982ebf..70e67cc145 100644 --- a/source/libs/tfs/test/CMakeLists.txt +++ b/source/libs/tfs/test/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries( PUBLIC gtest_main ) -#add_test( -# NAME tfs_test -# COMMAND tfs_test -#) +add_test( + NAME tfs_test + COMMAND tfs_test +) From 9f14c599f22db08532cd07bf8139eb3dfe5b24a4 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 10:49:27 +0800 Subject: [PATCH 41/60] fix: add TD_PATH_MAX to support multi-platform --- include/os/osDef.h | 10 ++++++++++ source/dnode/vnode/src/tsdb/tsdbOpen.c | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index 297d19e21a..3f4dcfc3c3 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -244,6 +244,16 @@ void syslog(int unused, const char *format, ...); #define TD_CHARSET_LEN 64 #define TD_TIMEZONE_LEN 96 +#if defined(WINDOWS) +#define TD_PATH_MAX MAX_PATH +#elif defined(PATH_MAX) +#define TD_PATH_MAX PATH_MAX +#elif defined(_XOPEN_PATH_MAX) +#define TD_PATH_MAX _XOPEN_PATH_MAX +#else +#define TD_PATH_MAX _POSIX_PATH_MAX +#endif + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index bc7021d603..5591d53ca9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = PATH_MAX; + slen = TD_PATH_MAX; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -48,7 +48,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } pTsdb->path = (char *)&pTsdb[1]; - snprintf(pTsdb->path, PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); + snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); From d42214e94af56cb48b9124bf5fba5a72929ea356 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 11:09:27 +0800 Subject: [PATCH 42/60] fix: add TD_PATH_MAX to support multi-platform --- include/os/osDef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index 3f4dcfc3c3..81c8a17ad1 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -245,7 +245,7 @@ void syslog(int unused, const char *format, ...); #define TD_TIMEZONE_LEN 96 #if defined(WINDOWS) -#define TD_PATH_MAX MAX_PATH +#define TD_PATH_MAX _MAX_PATH #elif defined(PATH_MAX) #define TD_PATH_MAX PATH_MAX #elif defined(_XOPEN_PATH_MAX) From 6547dbd2a7f41d4ffbcb5543ce281b3249ff5151 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 1 Nov 2022 11:30:44 +0800 Subject: [PATCH 43/60] fix: interp _irowts error --- source/libs/planner/src/planLogicCreater.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 74941d95f1..3a67390ee9 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -587,6 +587,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt return code; } +static bool isInterpFunc(int32_t funcId) { return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId); } + static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasInterpFunc) { return TSDB_CODE_SUCCESS; @@ -602,7 +604,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; // interp functions and _group_key functions - int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pInterpFunc->pFuncs); + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, isInterpFunc, &pInterpFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); } From 973a942d0fbfda4206be8adc331f534d63f7eeca Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 1 Nov 2022 11:43:40 +0800 Subject: [PATCH 44/60] enh: support get meta only from cache --- include/libs/catalog/catalog.h | 3 +++ source/libs/catalog/src/catalog.c | 30 ++++++++++++++++++++--- source/libs/catalog/src/ctgUtil.c | 5 ++++ source/libs/catalog/test/catalogTests.cpp | 20 +++++++++++++++ source/libs/qcom/src/queryUtil.c | 1 + source/libs/scalar/src/filter.c | 20 ++++++++------- source/libs/scalar/src/scalar.c | 4 +++ 7 files changed, 71 insertions(+), 12 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index dfeb68ce43..b957be4267 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -309,6 +309,9 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, + bool* pass, bool* exists); + int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b6a22b5fa7..e66cdb14ce 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -320,7 +320,7 @@ _return: } int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass) { + bool* pass, bool* exists) { bool inCache = false; int32_t code = 0; @@ -329,6 +329,13 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, co CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); if (inCache) { + if (exists) { + *exists = true; + } + + return TSDB_CODE_SUCCESS; + } else if (exists) { + *exists = false; return TSDB_CODE_SUCCESS; } @@ -1032,7 +1039,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; - strcpy(stb.tname, stbName); + tstrncpy(stb.tname, stbName, sizeof(stb.tname)); ctgRemoveTbMeta(pCtg, &stb); break; } @@ -1373,13 +1380,30 @@ int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass)); + CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, NULL)); _return: CTG_API_LEAVE(code); } +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, + bool* pass, bool* exists) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, exists)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 4a64a8666e..3d21bbbcd9 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -924,6 +924,11 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* if (1 == vgNum) { void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + if (NULL == pIter) { + ctgError("empty vgHash, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 0bdd9841ab..489d174e17 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2771,10 +2771,30 @@ TEST(apiTest, catalogChkAuth_test) { ASSERT_EQ(code, 0); bool pass = false; + bool exists = false; + code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + ASSERT_EQ(code, 0); + ASSERT_EQ(exists, false); + code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass); ASSERT_EQ(code, 0); ASSERT_EQ(pass, true); + while (true) { + uint64_t n = 0; + ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + if (n != 1) { + taosMsleep(50); + } else { + break; + } + } + + code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + ASSERT_EQ(code, 0); + ASSERT_EQ(pass, true); + ASSERT_EQ(exists, true); + catalogDestroy(); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 1a7965397e..cf064881c2 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -456,6 +456,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == (*pDst)->vgHash) { + taosMemoryFreeClear(*pDst); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 426274a20f..4738e1bbf9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1087,7 +1087,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - info->units = tmp; + info->units = (SFilterUnit*)tmp; memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); } @@ -1633,12 +1633,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SValueNode *var = (SValueNode *)field->desc; SDataType *dType = &var->node.resType; - if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { - qDebug("VAL%d => [type:TS][val:[%" PRIi64 "] - [%" PRId64 "]]", i, *(int64_t *)field->data, - *(((int64_t *)field->data) + 1)); - } else { + //if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { + // qDebug("VAL%d => [type:TS][val:[%" PRIi64 "] - [%" PRId64 "]]", i, *(int64_t *)field->data, + // *(((int64_t *)field->data) + 1)); + //} else { qDebug("VAL%d => [type:%d][val:%" PRIx64 "]", i, dType->type, var->datum.i); // TODO - } + //} } else if (field->data) { qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO } @@ -4059,11 +4059,13 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); - FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); - *p = output.columnData; - + int32_t code = scalarCalculate(info->sclCtx.node, pList, &output); taosArrayDestroy(pList); + FLT_ERR_RET(code); + + *p = output.columnData; + if (output.numOfQualified == output.numOfRows) { *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; } else if (output.numOfQualified == 0) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 70d1bb2685..ea1ce175e3 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -896,6 +896,10 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + if (NULL == pWhen || NULL == pThen) { + sclError("invalid when/then in whenThen list"); + SCL_ERR_JRET(TSDB_CODE_INVALID_PARA); + } if (pCase) { vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL); From 0b6d117121af89ab911c908ae9701b53cf06286b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 11:44:17 +0800 Subject: [PATCH 45/60] fix: add TD_PATH_MAX to support multi-platform --- include/os/osDef.h | 4 ++-- source/dnode/mnode/impl/test/db/CMakeLists.txt | 5 +++-- source/dnode/vnode/src/tsdb/tsdbOpen.c | 4 ++-- source/libs/tfs/test/CMakeLists.txt | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index 81c8a17ad1..07cd197ad7 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -244,8 +244,8 @@ void syslog(int unused, const char *format, ...); #define TD_CHARSET_LEN 64 #define TD_TIMEZONE_LEN 96 -#if defined(WINDOWS) -#define TD_PATH_MAX _MAX_PATH +#ifdef WINDOWS +#define TD_PATH_MAX 260 #elif defined(PATH_MAX) #define TD_PATH_MAX PATH_MAX #elif defined(_XOPEN_PATH_MAX) diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt index 04bd0d984d..ca8286b2f6 100644 --- a/source/dnode/mnode/impl/test/db/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -6,6 +6,7 @@ target_link_libraries( ) add_test( - NAME dbTest - COMMAND dbTest + NAME dbTest + COMMAND dbTest ) + diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index 5591d53ca9..a1012a26a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = TD_PATH_MAX; + slen = 260; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -48,7 +48,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } pTsdb->path = (char *)&pTsdb[1]; - snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); + snprintf(pTsdb->path, 260, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt index 70e67cc145..6b0f747147 100644 --- a/source/libs/tfs/test/CMakeLists.txt +++ b/source/libs/tfs/test/CMakeLists.txt @@ -9,6 +9,6 @@ target_link_libraries( ) add_test( - NAME tfs_test - COMMAND tfs_test + NAME tfs_test + COMMAND tfs_test ) From bba66861484d6822fcf88aa8182f6e784e42693e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 1 Nov 2022 11:48:07 +0800 Subject: [PATCH 46/60] test:modify case --- tests/system-test/7-tmq/tmqError.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 7ee9e5d169..5c858d0dce 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -244,7 +244,7 @@ class TDTestCase: tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 9000000 # Forever loop + paraDict['pollDelay'] = 9000000 # Forever loop showMsg = 1 showRow = 1 #self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) From 12db9c46242f4b6ced7e5b754e457c102947192a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Nov 2022 11:52:09 +0800 Subject: [PATCH 47/60] fix: taosbenchmark query concurrent segfault (#17795) update taos-tools 719fc88 --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 2b5994e95c..e3e12fa0b5 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 7321fbb + GIT_TAG 719fc88 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 5e253cc8ea1bf7438f5fafb095e1fbf9968d6c7b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 1 Nov 2022 11:56:14 +0800 Subject: [PATCH 48/60] refactor(stream): fill history --- include/libs/executor/executor.h | 1 + source/dnode/vnode/src/tq/tq.c | 13 +- source/libs/executor/inc/executorimpl.h | 13 +- source/libs/executor/src/executor.c | 104 +- source/libs/executor/src/scanoperator.c | 4 + source/libs/executor/src/timewindowoperator.c | 3 +- source/libs/stream/inc/streamInc.h | 3 + source/libs/stream/src/streamDispatch.c | 6 +- source/libs/stream/src/streamExec.c | 7 +- source/libs/stream/src/streamRecover.c | 28 +- tests/script/jenkins/basic.txt | 1 + .../script/tsim/stream/fillHistoryBasic1.sim | 926 ++++++++++++++++++ 12 files changed, 1065 insertions(+), 44 deletions(-) create mode 100644 tests/script/tsim/stream/fillHistoryBasic1.sim diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 3b1b63496f..1fe30a2d66 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -79,6 +79,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); */ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema); +int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); /** * Set multiple input data blocks for the stream scan. * @param tinfo diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 22ce4b20f5..47a40ea495 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1259,7 +1259,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { pSubmit = streamDataSubmitNew(pReq); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - qError("failed to create data submit for stream since out of memory"); + tqError("failed to create data submit for stream since out of memory"); failed = true; } @@ -1268,18 +1268,21 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__RECOVER1) continue; + if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__RECOVER1) { + tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); + continue; + } - qDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { - qError("stream task input failed, task id %d", pTask->taskId); + tqError("stream task input failed, task id %d", pTask->taskId); continue; } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + tqError("stream task launch failed, task id %d", pTask->taskId); continue; } } else { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8769e8ac2f..8b3b783b3f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -150,18 +150,13 @@ typedef struct { SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; - SSDataBlock* pullOverBlk; // for streaming - SWalFilterCond cond; - int64_t lastScanUid; int8_t recoverStep; SQueryTableDataCond tableCond; - int64_t recoverStartVer; - int64_t recoverEndVer; int64_t fillHistoryVer1; int64_t fillHistoryVer2; - int8_t triggerSaved; - int64_t deleteMarkSaved; + // int8_t triggerSaved; + // int64_t deleteMarkSaved; SStreamState* pState; } SStreamTaskInfo; @@ -461,8 +456,10 @@ typedef struct SPartitionDataInfo { typedef struct STimeWindowAggSupp { int8_t calTrigger; - int64_t waterMark; + int8_t calTriggerSaved; int64_t deleteMark; + int64_t deleteMarkSaved; + int64_t waterMark; TSKEY maxTs; TSKEY minTs; SColumnInfoData timeWindowData; // query time window info for scalar function execution. diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 9546c3895e..aae99c3286 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -70,6 +70,26 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } } +static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { + { + ASSERT(pOperator != NULL); + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (pOperator->numOfDownstream > 1) { // not handle this in join query + qError("join not supported for stream block scan, %s" PRIx64, id); + return TSDB_CODE_QRY_APP_ERROR; + } + pOperator->status = OP_NOT_OPENED; + return doSetStreamOpOpen(pOperator->pDownstream[0], id); + } + } + return 0; +} + static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -117,7 +137,22 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); } +int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { + if (tinfo == NULL) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + + int32_t code = doSetStreamOpOpen(pTaskInfo->pRoot, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); + } else { + qDebug("%s set the stream block successfully", GET_TASKID(pTaskInfo)); + } + + return code; +} int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { @@ -707,8 +742,7 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.recoverStartVer = 0; - pTaskInfo->streamInfo.recoverEndVer = ver; + pTaskInfo->streamInfo.fillHistoryVer1 = ver; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE1; return 0; } @@ -716,8 +750,7 @@ int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.recoverStartVer = pTaskInfo->streamInfo.recoverEndVer; - pTaskInfo->streamInfo.recoverEndVer = ver; + pTaskInfo->streamInfo.fillHistoryVer2 = ver; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE2; return 0; } @@ -738,22 +771,44 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - pTaskInfo->streamInfo.triggerSaved = pInfo->twAggSup.calTrigger; - pTaskInfo->streamInfo.deleteMarkSaved = pInfo->twAggSup.deleteMark; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0); + ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + + qInfo("save stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + + pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; + pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - pTaskInfo->streamInfo.triggerSaved = pInfo->twAggSup.calTrigger; - pTaskInfo->streamInfo.deleteMarkSaved = pInfo->twAggSup.deleteMark; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0); + ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + + qInfo("save stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + + pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; + pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; - pTaskInfo->streamInfo.triggerSaved = pInfo->twAggSup.calTrigger; - pTaskInfo->streamInfo.deleteMarkSaved = pInfo->twAggSup.deleteMark; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0); + ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + + qInfo("save stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + + pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; + pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; } @@ -783,21 +838,36 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX); - pInfo->twAggSup.calTrigger = pTaskInfo->streamInfo.triggerSaved; - pInfo->twAggSup.deleteMark = pTaskInfo->streamInfo.deleteMarkSaved; + pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; + pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX); - pInfo->twAggSup.calTrigger = pTaskInfo->streamInfo.triggerSaved; - pInfo->twAggSup.deleteMark = pTaskInfo->streamInfo.deleteMarkSaved; + pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; + pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX); - pInfo->twAggSup.calTrigger = pTaskInfo->streamInfo.triggerSaved; - pInfo->twAggSup.deleteMark = pTaskInfo->streamInfo.deleteMarkSaved; + pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; + pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } // iterate operator tree diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e2f3b1c6c4..1a1a622efd 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1841,6 +1841,10 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return pBlock; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + pTSInfo->cond.startVersion = 0; + pTSInfo->cond.endVersion = -1; + return NULL; } #endif diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 8e1b15f315..4da5bdd40e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2453,7 +2453,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } else { // non-linear interpolation pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - } } @@ -3302,6 +3301,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, // for test 315360000000 .deleteMark = 1000LL * 60LL * 60LL * 24LL * 365LL * 10LL, // .deleteMark = INT64_MAX, + .deleteMarkSaved = 0, + .calTriggerSaved = 0, }; ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index e9d4fbeaaa..a8f7184bb2 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -43,6 +43,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); +int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, + SEpSet* pEpSet); + SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); void streamFreeQitem(SStreamQueueItem* data); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 7cccda51a1..d2876a22c6 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -250,7 +250,7 @@ FAIL: return code; } -int32_t streamDispatchOneReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { +int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; @@ -371,7 +371,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, downstreamTaskId, vgId); - if (streamDispatchOneReq(pTask, &req, vgId, pEpSet) < 0) { + if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) { goto FAIL_FIXED_DISPATCH; } code = 0; @@ -433,7 +433,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (pReqs[i].blockNum > 0) { // send SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - if (streamDispatchOneReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { + if (streamDispatchOneDataReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { goto FAIL_SHUFFLE_DISPATCH; } } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6e37810004..629333b4b4 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -90,6 +90,8 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { void* exec = pTask->exec.executor; + qSetStreamOpOpen(exec); + while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -127,7 +129,10 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; streamTaskOutput(pTask, qRes); - // TODO stream sched dispatch + + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + streamDispatch(pTask); + } } return 0; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index c979ba058b..adeb797721 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -45,7 +45,6 @@ int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { // return streamScanExec(pTask, 100); - // TODO next: dispatch msg to launch scan step2 } int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { @@ -66,11 +65,20 @@ int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { SStreamRecoverFinishReq req = { .streamId = pTask->streamId, - .taskId = pTask->taskId, .childId = pTask->selfChildId, }; + // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + req.taskId = pTask->fixedEpDispatcher.taskId; + streamDispatchOneRecoverFinishReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); + for (int32_t i = 0; i < vgSz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + req.taskId = pVgInfo->taskId; + streamDispatchOneRecoverFinishReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + } } return 0; } @@ -78,9 +86,9 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { // agg int32_t streamAggRecoverPrepare(SStreamTask* pTask) { void* exec = pTask->exec.executor; - if (qStreamSetParamForRecover(exec) < 0) { - return -1; - } + /*if (qStreamSetParamForRecover(exec) < 0) {*/ + /*return -1;*/ + /*}*/ pTask->recoverWaitingChild = taosArrayGetSize(pTask->childEpInfo); return 0; } @@ -98,10 +106,12 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { } int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { - int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1); - ASSERT(left >= 0); - if (left == 0) { - streamAggChildrenRecoverFinish(pTask); + if (pTask->taskLevel == TASK_LEVEL__AGG) { + int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1); + ASSERT(left >= 0); + if (left == 0) { + streamAggChildrenRecoverFinish(pTask); + } } return 0; } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index cf3d26eb17..93a54b6c7d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -233,6 +233,7 @@ ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim +./test.sh -f tsim/stream/fillHistoryBasic1.sim ./test.sh -f tsim/stream/distributeInterval0.sim ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ./test.sh -f tsim/stream/distributeSession0.sim diff --git a/tests/script/tsim/stream/fillHistoryBasic1.sim b/tests/script/tsim/stream/fillHistoryBasic1.sim new file mode 100644 index 0000000000..5bbaf1b712 --- /dev/null +++ b/tests/script/tsim/stream/fillHistoryBasic1.sim @@ -0,0 +1,926 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database test vgroups 1; +sql select * from information_schema.ins_databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test; + +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream1 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sql insert into t1 values(1648791243003,4,2,3,3.1); +sql insert into t1 values(1648791213004,4,2,3,4.1); + + +sleep 1000 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 2 then + print ======$data13 + return -1 +endi + +if $data14 != 2 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223001,12,14,13,11.1); +sleep 500 +sql select * from streamt; + +print count(*) , count(d) , sum(a) , max(b) , min(c) +print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 +print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 12 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223002,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 2 then + print ======$data11 + return -1 +endi + +if $data12 != 2 then + print ======$data12 + return -1 +endi + +if $data13 != 24 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223003,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 36 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223001,1,1,1,1.1); +sql insert into t1 values(1648791223002,2,2,2,2.1); +sql insert into t1 values(1648791223003,3,3,3,3.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 6 then + print ======$data13 + return -1 +endi + +if $data14 != 3 then + print ======$data14 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791233003,3,2,3,2.1); +sql insert into t1 values(1648791233002,5,6,7,8.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 2 +if $data21 != 2 then + print ======$data21 + return -1 +endi + +if $data22 != 2 then + print ======$data22 + return -1 +endi + +if $data23 != 6 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 0 +if $data01 != 4 then + print ======$data01 + return -1 +endi + +if $data02 != 4 then + print ======$data02 + return -1 +endi + +if $data03 != 50 then + print ======$data03 != 50 + return -1 +endi + +if $data04 != 20 then + print ======$data04 != 20 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 4 then + print ======$data11 + return -1 +endi + +if $data12 != 4 then + print ======$data12 + return -1 +endi + +if $data13 != 46 then + print ======$data13 != 46 + return -1 +endi + +if $data14 != 20 then + print ======$data14 != 20 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 4 then + print ======$data21 + return -1 +endi + +if $data22 != 4 then + print ======$data22 + return -1 +endi + +if $data23 != 15 then + print ======$data23 + return -1 +endi + +if $data24 != 4 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + + + + + +sql create database test2 vgroups 1; +sql select * from information_schema.ins_databases; + +sql use test2; + +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sql insert into t1 values(1648791243003,4,2,3,3.1); +sql insert into t1 values(1648791213004,4,2,3,4.1); + +sql create stream stream2 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); + +sleep 1000 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 2 then + print ======$data13 + return -1 +endi + +if $data14 != 2 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223001,12,14,13,11.1); +sleep 500 +sql select * from streamt; + +print count(*) , count(d) , sum(a) , max(b) , min(c) +print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 +print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 12 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223002,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 2 then + print ======$data11 + return -1 +endi + +if $data12 != 2 then + print ======$data12 + return -1 +endi + +if $data13 != 24 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223003,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 36 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223001,1,1,1,1.1); +sql insert into t1 values(1648791223002,2,2,2,2.1); +sql insert into t1 values(1648791223003,3,3,3,3.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 6 then + print ======$data13 + return -1 +endi + +if $data14 != 3 then + print ======$data14 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791233003,3,2,3,2.1); +sql insert into t1 values(1648791233002,5,6,7,8.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 2 +if $data21 != 2 then + print ======$data21 + return -1 +endi + +if $data22 != 2 then + print ======$data22 + return -1 +endi + +if $data23 != 6 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 0 +if $data01 != 4 then + print ======$data01 + return -1 +endi + +if $data02 != 4 then + print ======$data02 + return -1 +endi + +if $data03 != 50 then + print ======$data03 != 50 + return -1 +endi + +if $data04 != 20 then + print ======$data04 != 20 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 4 then + print ======$data11 + return -1 +endi + +if $data12 != 4 then + print ======$data12 + return -1 +endi + +if $data13 != 46 then + print ======$data13 != 46 + return -1 +endi + +if $data14 != 20 then + print ======$data14 != 20 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 4 then + print ======$data21 + return -1 +endi + +if $data22 != 4 then + print ======$data22 + return -1 +endi + +if $data23 != 15 then + print ======$data23 + return -1 +endi + +if $data24 != 4 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + + From 5d853b11ab046e3c0b099698107f71c30f861cfa Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 1 Nov 2022 12:40:12 +0800 Subject: [PATCH 49/60] test:modify case --- tests/system-test/7-tmq/tmqShow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index d0b46bf572..0a21680e78 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -138,8 +138,9 @@ class TDTestCase: tdSql.query("show subscriptions") # tdLog.debug(tdSql.queryResult) rows = tdSql.getRows() - tdLog.info("show subscriptions rows: %d"%rows) - if rows != paraDict['vgroups'] * len(topicNameList): + expectSubscriptions = paraDict['vgroups'] * len(topicNameList) + tdLog.info("show subscriptions rows: %d, expect Subscriptions: %d"%(rows,expectSubscriptions)) + if rows != expectSubscriptions: tdLog.exit("show subscriptions rows error") pThread.join() From 1b179f65b2fccb50b126315979e45680eb5d9873 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 1 Nov 2022 13:34:10 +0800 Subject: [PATCH 50/60] fix: interp _irowts error --- include/libs/function/functionMgt.h | 1 + source/libs/function/src/functionMgt.c | 7 +++++++ source/libs/planner/src/planLogicCreater.c | 4 +++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 6968f1712c..a50af18b44 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -216,6 +216,7 @@ bool fmIsMultiRowsFunc(int32_t funcId); bool fmIsKeepOrderFunc(int32_t funcId); bool fmIsCumulativeFunc(int32_t funcId); bool fmIsInterpPseudoColumnFunc(int32_t funcId); +bool fmIsGroupKeyFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index bae005f5c4..0de914271c 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -251,6 +251,13 @@ bool fmIsSelectValueFunc(int32_t funcId) { return FUNCTION_TYPE_SELECT_VALUE == funcMgtBuiltins[funcId].type; } +bool fmIsGroupKeyFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_GROUP_KEY == funcMgtBuiltins[funcId].type; +} + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 3a67390ee9..275983f869 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -587,7 +587,9 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt return code; } -static bool isInterpFunc(int32_t funcId) { return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId); } +static bool isInterpFunc(int32_t funcId) { + return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId); +} static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasInterpFunc) { From 5ff701ef1d5a6ce1d3b8fed70e10dbff58cfef0f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 1 Nov 2022 13:35:23 +0800 Subject: [PATCH 51/60] fix(query):reset tsdb reader --- source/libs/executor/src/scanoperator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eba3dd892d..66cb960a57 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1875,6 +1875,11 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; } + + /*resetTableScanInfo(pTSInfo, pWin);*/ + tsdbReaderClose(pTSInfo->dataReader); + pTSInfo->dataReader = NULL; + pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; From 7a8c2efbc3330212caffd35a3159dafc3ee3bcb4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Nov 2022 13:44:04 +0800 Subject: [PATCH 52/60] fix: add jemalloc as build dependency of taosd, udf and shell (#17785) * fix: jemalloc compile error * fix: jemalloc compile error * fix: add jemalloc as dependency of taosd and udf Co-authored-by: afwerar <1296468573@qq.com> --- source/dnode/mgmt/CMakeLists.txt | 16 +++++++++- source/libs/function/CMakeLists.txt | 48 +++++++++++++++++++++++------ tools/shell/CMakeLists.txt | 10 +++++- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 423028b167..762b8fd529 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -13,4 +13,18 @@ target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) -target_link_libraries(taosd dnode) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(taosd jemalloc) + target_link_libraries(taosd dnode ${LINK_JEMALLOC}) +ELSE () + target_link_libraries(taosd dnode) +ENDIF () + diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 913dd24a49..fa241dc6ef 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -12,6 +12,17 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(function jemalloc) +ENDIF () + target_link_libraries( function PRIVATE os @@ -21,7 +32,7 @@ target_link_libraries( PRIVATE qcom PRIVATE scalar PRIVATE transport - PRIVATE stream + PRIVATE stream ${LINK_JEMALLOC} PUBLIC uv_a ) @@ -37,10 +48,15 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(runUdf jemalloc) +ENDIF () + target_link_libraries( runUdf PUBLIC uv_a - PRIVATE os util common nodes function + PRIVATE os util common nodes function ${LINK_JEMALLOC} ) add_library(udf1 STATIC MODULE test/udf1.c) @@ -54,8 +70,13 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf1 jemalloc) +ENDIF () + target_link_libraries( - udf1 PUBLIC os) + udf1 PUBLIC os ${LINK_JEMALLOC}) add_library(udf2 STATIC MODULE test/udf2.c) target_include_directories( @@ -68,8 +89,13 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf2 jemalloc) +ENDIF () + target_link_libraries( - udf2 PUBLIC os + udf2 PUBLIC os ${LINK_JEMALLOC} ) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) @@ -86,9 +112,13 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - udfd - PUBLIC uv_a - PRIVATE os util common nodes function -) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udfd jemalloc) +ENDIF () + +target_link_libraries( + udfd + PUBLIC uv_a + PRIVATE os util common nodes function ${LINK_JEMALLOC} + ) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 31dcde036d..1e7d0ed140 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -2,6 +2,14 @@ aux_source_directory(src SHELL_SRC) add_executable(shell ${SHELL_SRC}) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") + ADD_DEPENDENCIES(shell jemalloc) +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + IF (TD_LINUX AND TD_WEBSOCKET) ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include -ltaosws) SET(LINK_WEBSOCKET "-L${CMAKE_BINARY_DIR}/build/lib -ltaosws") @@ -21,7 +29,7 @@ ENDIF () if(TD_WINDOWS) target_link_libraries(shell PUBLIC taos_static ${LINK_WEBSOCKET}) else() - target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET}) + target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET} ${LINK_JEMALLOC}) endif () target_link_libraries( shell From 83495ed5d3a22e2fe6209cf3456f1e17370bb934 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Nov 2022 13:49:02 +0800 Subject: [PATCH 53/60] fix(query): set the correct buffer to accommodate the tag values. --- source/common/src/tdatablock.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbRead.c | 8 ++------ source/libs/executor/src/executorimpl.c | 4 ++++ source/libs/executor/src/scanoperator.c | 10 +++++++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3afa91f2f9..86b851877f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1132,6 +1132,10 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.window.ekey = 0; pDataBlock->info.window.skey = 0; + if (pDataBlock->info.capacity == 0) { + return; + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2b3eff0297..2967a96453 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3837,7 +3837,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - int64_t stime = taosGetTimestampUs(); +// int64_t stime = taosGetTimestampUs(); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; @@ -3887,14 +3887,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } } - double elapsed = (taosGetTimestampUs() - stime) / 1000.0; - pReader->cost.smaLoadTime += elapsed; pReader->cost.smaDataLoad += 1; - *pBlockStatis = pSup->plist; - tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", elapsed time:%.2f ms, %s", 0, pFBlock->uid, - elapsed, pReader->idStr); + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5b26005a19..286885d018 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -574,6 +574,9 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // if the source equals to the destination, it is to create a new column as the result of scalar // function or some operators. bool createNewColModel = (pResult == pSrcBlock); + if (createNewColModel) { + blockDataEnsureCapacity(pResult, pResult->info.rows); + } int32_t numOfRows = 0; @@ -623,6 +626,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; ASSERT(pResult->info.capacity > 0); + colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows); colDataDestroy(&idata); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b3226f292a..c731823392 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -424,8 +424,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - // NOTE: here the tag value only load for one row - ensureBlockCapacity(pBlock, 1); + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + ensureBlockCapacity(pBlock, pBlock->info.rows); + } doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; @@ -436,7 +437,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - ensureBlockCapacity(pBlock, 1); + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + ensureBlockCapacity(pBlock, pBlock->info.rows); + } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); return TSDB_CODE_SUCCESS; } else { From f4e7f3da5e8bb8a7f8a509f7ab23864776f8fbb3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Nov 2022 14:05:54 +0800 Subject: [PATCH 54/60] test: comment out unstable case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 6328124b0d..f1340a6968 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -321,7 +321,7 @@ # --- vnode ---- ./test.sh -f tsim/vnode/replica3_basic.sim -./test.sh -f tsim/vnode/replica3_repeat.sim +# TD-20089 ./test.sh -f tsim/vnode/replica3_repeat.sim ./test.sh -f tsim/vnode/replica3_vgroup.sim ./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/vnode/replica3_import.sim From a28e9284e0c8c6cfeeec140016ae6008a6d35855 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Nov 2022 14:13:00 +0800 Subject: [PATCH 55/60] test: adjust unstable case --- tests/script/jenkins/basic.txt | 2 +- tests/script/tsim/vnode/replica3_repeat.sim | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f1340a6968..6328124b0d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -321,7 +321,7 @@ # --- vnode ---- ./test.sh -f tsim/vnode/replica3_basic.sim -# TD-20089 ./test.sh -f tsim/vnode/replica3_repeat.sim +./test.sh -f tsim/vnode/replica3_repeat.sim ./test.sh -f tsim/vnode/replica3_vgroup.sim ./test.sh -f tsim/vnode/replica3_many.sim ./test.sh -f tsim/vnode/replica3_import.sim diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index 4902cde7c5..ccff06819b 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -107,7 +107,16 @@ system sh/exec.sh -n dnode2 -s stop sleep 3000 print ======== step7 -sql select count(*) from db.tb +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 30 then + print ====> dnode not ready! + return -1 + endi +sql select count(*) from db.tb -x step7 + print select count(*) from db.tb ==> $data00 $lastRows if $data00 <= $lastRows then return -1 From 65943b3712c663f213c4fcc82276ae70fd84e54e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 14:51:37 +0800 Subject: [PATCH 56/60] chore: revert the code change --- source/dnode/mnode/impl/test/db/CMakeLists.txt | 1 - source/dnode/vnode/src/tsdb/tsdbOpen.c | 4 ++-- source/libs/tfs/test/CMakeLists.txt | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/test/db/CMakeLists.txt b/source/dnode/mnode/impl/test/db/CMakeLists.txt index ca8286b2f6..3f6a80835f 100644 --- a/source/dnode/mnode/impl/test/db/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/db/CMakeLists.txt @@ -9,4 +9,3 @@ add_test( NAME dbTest COMMAND dbTest ) - diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index a1012a26a6..efc74b68ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -38,7 +38,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee int slen = 0; *ppTsdb = NULL; - slen = 260; + slen = strlen(pVnode->path) + strlen(dir) + 2; // create handle pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen); @@ -48,7 +48,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } pTsdb->path = (char *)&pTsdb[1]; - snprintf(pTsdb->path, 260, "%s%s%s", pVnode->path, TD_DIRSEP, dir); + sprintf(pTsdb->path, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); diff --git a/source/libs/tfs/test/CMakeLists.txt b/source/libs/tfs/test/CMakeLists.txt index 6b0f747147..ee28dcf723 100644 --- a/source/libs/tfs/test/CMakeLists.txt +++ b/source/libs/tfs/test/CMakeLists.txt @@ -9,6 +9,6 @@ target_link_libraries( ) add_test( - NAME tfs_test - COMMAND tfs_test + NAME tfs_test + COMMAND tfs_test ) From cff5d9295e06a17085b40f2954c6fd82751b408a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Nov 2022 15:00:02 +0800 Subject: [PATCH 57/60] refactor: add some logs. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 28 ++++++----- source/libs/executor/inc/executil.h | 3 +- source/libs/executor/inc/executorimpl.h | 21 ++------- source/libs/executor/src/executil.c | 10 ++-- source/libs/executor/src/executorimpl.c | 62 +++++++++---------------- 5 files changed, 49 insertions(+), 75 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f90c567be..b6e8362092 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -72,6 +72,7 @@ typedef struct SIOCostSummary { double lastBlockLoadTime; int64_t composedBlocks; double buildComposedBlockTime; + double createScanInfoList; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -234,6 +235,8 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK return NULL; } + int64_t st = taosGetTimestampUs(); + for (int32_t j = 0; j < numOfTables; ++j) { STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { @@ -249,8 +252,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK pTsdbReader->idStr); } - tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, %s", pTsdbReader, numOfTables, - (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->idStr); + pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables, + (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, + pTsdbReader->idStr); return pTableMap; } @@ -3697,15 +3702,16 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } - tsdbDebug( - "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 - ", fileBlocks-load-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 - ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, - pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, - pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); + tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 + ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 + ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, creatTime:%.2f ms, %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, + pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, + pCost->lastBlockLoadTime, pCost->composedBlocks, pCost->buildComposedBlockTime, + numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 76a3359b5b..fd8a357a8b 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -96,11 +96,12 @@ typedef struct SColMatchInfo { int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; +typedef struct SExecTaskInfo SExecTaskInfo; typedef struct STableListInfo STableListInfo; struct SqlFunctionCtx; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* id); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo); STableListInfo* tableListCreate(); void* tableListDestroy(STableListInfo* pTableListInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1191b6a485..c9703e396f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -89,25 +89,10 @@ typedef struct STableScanAnalyzeInfo SFileBlockLoadRecorder; typedef struct STaskCostInfo { int64_t created; int64_t start; - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; - - uint64_t loadDataTime; - + uint64_t elapsedTime; + double extractListTime; + double groupIdMapTime; SFileBlockLoadRecorder* pRecoder; - uint64_t elapsedTime; - - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; - - SArray* queryProfEvents; // SArray - SHashObj* operatorProfResults; // map } STaskCostInfo; typedef struct SOperatorCostInfo { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index bb87b5029a..8bdc110ee5 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1882,8 +1882,9 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, - const char* idStr) { + struct SExecTaskInfo* pTaskInfo) { int64_t st = taosGetTimestampUs(); + const char* idStr = GET_TASKID(pTaskInfo); if (pHandle == NULL) { qError("invalid handle, in creating operator tree, %s", idStr); @@ -1899,7 +1900,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags ASSERT(pTableListInfo->numOfOuputGroups == 1); int64_t st1 = taosGetTimestampUs(); - qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1 - st) / 1000.0, idStr); + pTaskInfo->cost.extractListTime = (st1 - st) / 1000.0; + qDebug("extract queried table list completed, elapsed time:%.2f ms %s", pTaskInfo->cost.extractListTime, idStr); if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { qDebug("no table qualified for query, %s" PRIx64, idStr); @@ -1911,8 +1913,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } - int64_t st2 = taosGetTimestampUs(); - qDebug("generate group id map completed, elapsed time:%.2f ms %s", (st2 - st1) / 1000.0, idStr); + pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1)/1000.0; + qDebug("generate group id map completed, elapsed time:%.2f ms %s", pTaskInfo->cost.groupIdMapTime, idStr); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0c0d5549f2..42bbbcbf88 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1447,40 +1447,18 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { - int32_t colSize = pColRes->info.bytes * numOfRows; - return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, - colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); -} - void queryCostStatis(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; - // uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - // hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); - // pSummary->hashSize = hashSize; - - // SResultRowPool* p = pTaskInfo->pool; - // if (p != NULL) { - // pSummary->winInfoSize = getResultRowPoolMemSize(p); - // pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); - // } else { - // pSummary->winInfoSize = 0; - // pSummary->numOfTimeWindows = 0; - // } - SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: elapsed time:%.2f ms, total blocks:%d, load block SMA:%d, load data block:%d, total " - "rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pRecorder->totalBlocks, pRecorder->loadBlockStatis, - pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); + "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " + "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, + pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, + pRecorder->totalCheckedRows); } - - // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, - // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, - // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } // static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -3267,6 +3245,11 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); + if (pTaskInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->schemaInfo.dbname = strdup(dbFName); @@ -3397,7 +3380,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, - pTableListInfo, pTagCond, pTagIndexCond, idstr); + pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr); @@ -3415,9 +3398,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - int32_t code = - createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, /*pTableScanNode->groupSort*/true, pHandle, - pTableListInfo, pTagCond, pTagIndexCond, idstr); + + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, + pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); @@ -3442,7 +3425,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, - pHandle, pTableListInfo, pTagCond, pTagIndexCond, idstr); + pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); @@ -3469,7 +3452,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, - pTagIndexCond, idstr); + pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; qError("failed to getTableList, code: %s", tstrerror(code)); @@ -3502,7 +3485,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - pTagCond, pTagIndexCond, idstr); + pTagCond, pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -3834,10 +3817,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead char* sql, EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; - int32_t code = TSDB_CODE_SUCCESS; *pTaskInfo = createExecTaskInfo(queryId, taskId, model, pPlan->dbFName); if (*pTaskInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; } @@ -3856,17 +3837,16 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { - code = (*pTaskInfo)->code; + terrno = (*pTaskInfo)->code; goto _complete; } - return code; + return TSDB_CODE_SUCCESS; - _complete: +_complete: taosMemoryFree(sql); doDestroyTask(*pTaskInfo); - terrno = code; - return code; + return terrno; } void doDestroyTask(SExecTaskInfo* pTaskInfo) { From 872085bef4ebe05c5b5a6ceb07cd99d2a2329c10 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 1 Nov 2022 15:18:26 +0800 Subject: [PATCH 58/60] fix: add TD_PATH_MAX to support multi-platform --- source/dnode/vnode/src/tsdb/tsdbOpen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index efc74b68ba..bb20a9b012 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -48,7 +48,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee } pTsdb->path = (char *)&pTsdb[1]; - sprintf(pTsdb->path, "%s%s%s", pVnode->path, TD_DIRSEP, dir); + snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir); taosRealPath(pTsdb->path, NULL, slen); pTsdb->pVnode = pVnode; taosThreadRwlockInit(&pTsdb->rwLock, NULL); From 20539ae04b50bf344ecccc1b439b0506a9a7662c Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Tue, 1 Nov 2022 15:45:55 +0800 Subject: [PATCH 59/60] Update tmqDnodeRestart1.py --- tests/system-test/7-tmq/tmqDnodeRestart1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 054afe4fef..714f10b362 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) From 638eaf2c58a1e980265a989b02b369b68fe2a614 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Nov 2022 16:14:37 +0800 Subject: [PATCH 60/60] fix: taosbenchmark free null child tbl name (#17801) * fix: taosbenchmark query concurrent segfault update taos-tools 719fc88 * fix: taosbenchmark free null child tbl name --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index e3e12fa0b5..83b7ade407 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 719fc88 + GIT_TAG 16eb34f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE