From a8dc0a9cb2f03eb3dd602fbe999557de74d1b693 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 27 Oct 2022 10:01:15 +0800 Subject: [PATCH 01/46] fix:add test case for schemaless --- utils/test/c/sml_test.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 83dfa1fc57..40d5bb12d2 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1097,18 +1097,12 @@ int sml_time_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - char* tmp = (char*)taosMemoryCalloc(1024, 1); - memcpy(tmp, sql[0], strlen(sql[0])); - *(char*)(tmp+44) = 0; - int32_t totalRows = 0; - pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT(totalRows == 3); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); taos_close(taos); - taosMemoryFree(tmp); return code; } From 9bf3b79420ddfd3536e34af8bef0217bca705762 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 10 Nov 2022 09:49:34 +0800 Subject: [PATCH 02/46] fix: disable sql log in schemaless if sql is too long --- source/client/src/clientSml.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index d1aeaac587..4cd1b5416c 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2082,7 +2082,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); + uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql", info->id); int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { From 1ca983917068cc46bb3c545a03a1c1064ffc92db Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 10 Nov 2022 15:47:07 +0800 Subject: [PATCH 03/46] fix(operator):Operator double free --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 5 +-- source/libs/executor/src/scanoperator.c | 4 +- source/libs/executor/src/tfill.c | 2 +- tests/script/tsim/stream/drop_stream.sim | 54 ++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 62146b6048..e1db1f4729 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -981,6 +981,7 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo); void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); +void destroyOperatorInfo(SOperatorInfo* pOperator); int32_t getMaximumIdleDurationSec(); /* diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 709e981a1f..34f462cb3d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -91,9 +91,6 @@ static void destroyAggOperatorInfo(void* param); static void destroyIntervalOperatorInfo(void* param); - -static void destroyOperatorInfo(SOperatorInfo* pOperator); - void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); @@ -2172,7 +2169,7 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { } } -static void destroyOperatorInfo(SOperatorInfo* pOperator) { +void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b83fac9fa6..2ea95b8dca 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2366,9 +2366,7 @@ _end: static void destroyStreamScanOperatorInfo(void* param) { SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { - STableScanInfo* pTableScanInfo = pStreamScan->pTableScanOp->info; - destroyTableScanOperatorInfo(pTableScanInfo); - taosMemoryFreeClear(pStreamScan->pTableScanOp); + destroyOperatorInfo(pStreamScan->pTableScanOp); } if (pStreamScan->tqReader) { tqCloseReader(pStreamScan->tqReader); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 7c9d73ad13..ddd948a6dd 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -680,9 +680,9 @@ SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) { void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) { for (int32_t i = start; i < end; i++) { destroyExprInfo(pFillCol[i].pExpr, 1); - taosMemoryFreeClear(pFillCol[i].pExpr); taosVariantDestroy(&pFillCol[i].fillVal); } + taosMemoryFreeClear(pFillCol[start].pExpr); taosMemoryFree(pFillCol); return NULL; } diff --git a/tests/script/tsim/stream/drop_stream.sim b/tests/script/tsim/stream/drop_stream.sim index b25e002140..817780ca59 100644 --- a/tests/script/tsim/stream/drop_stream.sim +++ b/tests/script/tsim/stream/drop_stream.sim @@ -216,6 +216,60 @@ sql insert into scalar_tb values (1656668180503+1s, -50, 50.1, "beiJing", "TDeng print ========== step6 repeat sql drop database test; + +print ========== interval\session\state window + +sql CREATE DATABASE test1 BUFFER 96 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 STRICT 'off' WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0; +sql use test1; +sql CREATE STABLE st (time TIMESTAMP, ca DOUBLE, cb DOUBLE, cc int) TAGS (ta VARCHAR(10) ); + +print ========== create table before stream + +sql CREATE TABLE t1 using st TAGS ('aaa'); +sql CREATE TABLE t2 using st TAGS ('bbb'); +sql CREATE TABLE t3 using st TAGS ('ccc'); +sql CREATE TABLE t4 using st TAGS ('ddd'); + +print ========== stable + +sql create stream streamd1 into streamt1 as select ca, _wstart,_wend, count(*) as total from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca interval(60m) fill(null); +sql create stream streamd2 into streamt2 as select ca, _wstart,_wend, count(*), max(ca), max(cb) from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca interval(60m) fill(linear); +sql create stream streamd3 into streamt3 as select ca, _wstart,_wend, count(*) as total from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca session(time, 60m); +sql create stream streamd4 into streamt4 as select ta, _wstart,_wend, count(*) as total from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ta session(time, 60m); +sql_error create stream streamd5 into streamt5 as select ca, _wstart,_wend, count(*) as total from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc); +sql_error create stream streamd6 into streamt6 as select ta, _wstart,_wend, count(*) as total from st where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ta state_window(cc); + +print ========== table + +sql create stream streamd7 into streamt7 as select ca, _wstart,_wend, count(*) as total from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca interval(60m) fill(null); +sql create stream streamd8 into streamt8 as select ca, _wstart,_wend, count(*), max(ca), max(cb) from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca interval(60m) fill(linear); +sql create stream streamd9 into streamt9 as select ca, _wstart,_wend, count(*) as total from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca session(time, 60m); +sql create stream streamd10 into streamt10 as select ta, _wstart,_wend, count(*) as total from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ta session(time, 60m); +sql create stream streamd11 into streamt11 as select ca, _wstart,_wend, count(*) as total from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ca state_window(cc); +sql create stream streamd12 into streamt12 as select ta, _wstart,_wend, count(*) as total from t1 where time > "2022-01-01 00:00:00" and time < "2032-01-01 00:00:00" partition by ta state_window(cc); + +print ========== create table after stream +sql CREATE TABLE t5 using st TAGS ('eee'); +sql CREATE TABLE t6 using st TAGS ('fff'); +sql CREATE TABLE t7 using st TAGS ('ggg'); +sql CREATE TABLE t8 using st TAGS ('fff'); + +sleep 1000 +print ========== drop stream +sql drop stream if exists streamd1; +sql drop stream if exists streamd2; +sql drop stream if exists streamd3; +sql drop stream if exists streamd4; +#sql drop stream if exists streamd5; +#sql drop stream if exists streamd6; +sql drop stream if exists streamd7; +sql drop stream if exists streamd8; +sql drop stream if exists streamd9; +sql drop stream if exists streamd10; +sql drop stream if exists streamd11; +sql drop stream if exists streamd12; +print ========== step7 + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT From 2481f1cdb37904dfc0a2de2cea103e7abb24ea2d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 10 Nov 2022 16:27:17 +0800 Subject: [PATCH 04/46] fix(shell): describe show command need show whole result --- tools/shell/src/shellEngine.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9bb02159f0..82550fb4e9 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -540,11 +540,20 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t } } -bool shellIsLimitQuery(const char *sql) { - // todo refactor +// show whole result for this query return true, like limit or describe +bool shellIsShowWhole(const char *sql) { + // limit if (taosStrCaseStr(sql, " limit ") != NULL) { return true; } + // describe + if (taosStrCaseStr(sql, "describe ") != NULL) { + return true; + } + // describe + if (taosStrCaseStr(sql, "show ") != NULL) { + return true; + } return false; } @@ -578,7 +587,7 @@ int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { uint64_t resShowMaxNum = UINT64_MAX; - if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsLimitQuery(sql)) { + if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } @@ -723,7 +732,7 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { uint64_t resShowMaxNum = UINT64_MAX; - if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsLimitQuery(sql)) { + if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } From 84a1105be01405e2ced5f692cf29ad9b845af55b Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 10 Nov 2022 16:48:24 +0800 Subject: [PATCH 05/46] fix(shell): describe show command need show whole result1 --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 82550fb4e9..8402a5a589 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -550,7 +550,7 @@ bool shellIsShowWhole(const char *sql) { if (taosStrCaseStr(sql, "describe ") != NULL) { return true; } - // describe + // show if (taosStrCaseStr(sql, "show ") != NULL) { return true; } From 5c5d70e0fc5844f71f82cc1bd81f82fb26a5ba87 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 10 Nov 2022 17:05:53 +0800 Subject: [PATCH 06/46] enh log --- source/libs/transport/src/transCli.c | 8 ++++++++ source/util/src/tlog.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 457d3e5cb1..b91875e84b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -407,9 +407,16 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { bool once = false; do { SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); + if (pMsg == NULL && once) { break; } + + if (pMsg != NULL && REQUEST_NO_RESP(&pMsg->msg)) { + destroyCmsg(pMsg); + break; + } + STransConnCtx* pCtx = pMsg ? pMsg->ctx : NULL; STransMsg transMsg = {0}; @@ -439,6 +446,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { continue; } } + if (pMsg == NULL || (pMsg && pMsg->type != Release)) { if (cliAppCb(pConn, &transMsg, pMsg) != 0) { return; diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index f7949048ca..a2ce5ac08c 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -435,7 +435,7 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { taosGetTimeOfDay(&timeSecs); time_t curTime = timeSecs.tv_sec; - ptm = taosLocalTimeNolock(&Tm, &curTime, taosGetDaylight()); + ptm = taosLocalTime(&curTime, &Tm); return sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " %s", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId(), flags); From 1c519bcae0068607c319de7c80b9bbaa525f2987 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 10 Nov 2022 18:43:57 +0800 Subject: [PATCH 07/46] redefine timer --- source/libs/transport/inc/transComm.h | 2 +- source/libs/transport/src/transCli.c | 36 +++++++++++++++------------ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 2354f0f959..9dcc174f9b 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -96,7 +96,7 @@ typedef void* queue[2]; #define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit #define TRANS_RETRY_INTERVAL 15 // retry interval (ms) -#define TRANS_CONN_TIMEOUT 3 // connect timeout (s) +#define TRANS_CONN_TIMEOUT 3000 // connect timeout (s) #define TRANS_READ_TIMEOUT 3000 // read timeout (ms) #define TRANS_PACKET_LIMIT 1024 * 1024 * 512 diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b91875e84b..3af0747bcf 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -25,7 +25,9 @@ typedef struct SCliConn { uv_connect_t connReq; uv_stream_t* stream; queue wreqQueue; - uv_timer_t* timer; + + uv_timer_t* timer; // read timer, forbidden + uv_timer_t connTimer; void* hostThrd; @@ -102,6 +104,8 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port); static void addConnToPool(void* pool, SCliConn* conn); static void doCloseIdleConn(void* param); +// register conn timer +static void cliConnTimeout(uv_timer_t* handle); // register timer for read static void cliReadTimeoutCb(uv_timer_t* handle); // register timer in each thread to clear expire conn @@ -462,6 +466,12 @@ void cliHandleExcept(SCliConn* conn) { cliHandleExceptImpl(conn, -1); } +void cliConnTimeout(uv_timer_t* handle) { + SCliConn* conn = handle->data; + tTrace("%s conn %p conn timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + uv_timer_stop(handle); + cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); +} void cliReadTimeoutCb(uv_timer_t* handle) { // set up timeout cb SCliConn* conn = handle->data; @@ -638,8 +648,10 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; - conn->connReq.data = conn; + uv_timer_init(pThrd->loop, &conn->connTimer); + conn->connTimer.data = conn; + conn->connReq.data = conn; transReqQueueInit(&conn->wreqQueue); transQueueInit(&conn->cliMsgs, NULL); @@ -819,6 +831,8 @@ _RETURN: void cliConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; + uv_timer_stop(&pConn->connTimer); + if (status != 0) { tError("%s conn %p failed to connect server:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_strerror(status)); cliHandleExcept(pConn); @@ -997,31 +1011,21 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet)); conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet); - int ret = transSetConnOption((uv_tcp_t*)conn->stream); - if (ret) { - tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret)); - } - int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT); - if (fd == -1) { - tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn); - cliHandleExcept(conn); - return; - } - uv_tcp_open((uv_tcp_t*)conn->stream, fd); - struct sockaddr_in addr; addr.sin_family = AF_INET; - addr.sin_addr.s_addr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, conn->ip); addr.sin_port = (uint16_t)htons((uint16_t)conn->port); tTrace("%s conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); - ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); + + int ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { tTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, uv_err_name(ret)); + uv_timer_stop(&conn->connTimer); cliHandleExcept(conn); return; } + uv_timer_start(&conn->connTimer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0); } STraceId* trace = &pMsg->msg.info.traceId; tGTrace("%s conn %p ready", pTransInst->label, conn); From 4d09254e93b1b4587c2a20b4952545905b1e1393 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Thu, 10 Nov 2022 21:06:20 +0800 Subject: [PATCH 08/46] test(driver):C# CI failed for longarch64 pr-17982 (#18030) * Add loongarch64 architecture port. Signed-off-by: Jingyun Hua * test(driver):C# CI failed for longarch64 pr-17982 Signed-off-by: Jingyun Hua Co-authored-by: Jingyun Hua --- cmake/cmake.platform | 12 ++++++++++++ docs/en/14-reference/09-support-platform/index.md | 2 +- docs/zh/14-reference/09-support-platform/index.md | 2 +- packaging/docker/dockerbuild.sh | 6 +++--- packaging/docker/dockerbuildi.sh | 4 ++-- packaging/release.sh | 8 ++++---- source/os/src/osSysinfo.c | 6 +++++- source/util/src/tcrc32c.c | 10 +++++----- tests/docs-examples-test/csharp.sh | 8 ++++---- 9 files changed, 37 insertions(+), 21 deletions(-) diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 3e239d2e0c..eb8b63b4c2 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -102,6 +102,12 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_ARM_64 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64") + MESSAGE(STATUS "The current platform is loongarch64") + SET(PLATFORM_ARCH_STR "loongarch64") + SET(TD_LOONGARCH_64 TRUE) + ADD_DEFINITIONS("-D_TD_LOONGARCH_") + ADD_DEFINITIONS("-D_TD_LOONGARCH_64") ENDIF () ELSE () # if generate ARM version: @@ -118,6 +124,12 @@ ELSE () ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_64") SET(TD_ARM_64 TRUE) + ELSEIF (${CPUTYPE} MATCHES "loongarch64") + SET(PLATFORM_ARCH_STR "loongarch64") + MESSAGE(STATUS "input cpuType: loongarch64") + SET(TD_LOONGARCH_64 TRUE) + ADD_DEFINITIONS("-D_TD_LOONGARCH_") + ADD_DEFINITIONS("-D_TD_LOONGARCH_64") ELSEIF (${CPUTYPE} MATCHES "mips64") SET(PLATFORM_ARCH_STR "mips") MESSAGE(STATUS "input cpuType: mips64") diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index fe26860765..061294f016 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -14,7 +14,7 @@ Note: ● means officially tested and verified, ○ means unofficially tested an ## List of supported platforms for TDengine clients and connectors -TDengine's connector can support a wide range of platforms, including X64/X86/ARM64/ARM32/MIPS/Alpha hardware platforms and Linux/Win64/Win32/macOS development environments. +TDengine's connector can support a wide range of platforms, including X64/X86/ARM64/ARM32/MIPS/Alpha/LoongArch64 hardware platforms and Linux/Win64/Win32/macOS development environments. The comparison matrix is as follows. diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index 7292ca4814..500eeeb14c 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -16,7 +16,7 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 客户端和连接器支持的平台列表 -目前 TDengine 的连接器可支持的平台广泛,目前包括:X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台,以及 Linux/Win64/Win32/macOS 等开发环境。 +目前 TDengine 的连接器可支持的平台广泛,目前包括:X64/X86/ARM64/ARM32/MIPS/LoongArch64 等硬件平台,以及 Linux/Win64/Win32/macOS 等开发环境。 对照矩阵如下: diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 4c8f1413aa..b02387a3d1 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -5,7 +5,7 @@ set -e #set -x # dockerbuild.sh -# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 | loongarch64...] # -n [version number] # -p [password for docker hub] # -V [stable | beta] @@ -57,7 +57,7 @@ do dockerLatest=$(echo $OPTARG) ;; h) - echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 | loongarch64...] " echo " -n [version number] " echo " -p [password for docker hub] " echo " -V [stable | beta] " @@ -136,4 +136,4 @@ if [ "$cloudBuild" != "y" ] && [ ${dockerLatest} == 'y' ] ;then docker push tdengine/tdengine-${dockername}:latest fi -rm -f ${pkgFile} \ No newline at end of file +rm -f ${pkgFile} diff --git a/packaging/docker/dockerbuildi.sh b/packaging/docker/dockerbuildi.sh index a0a954e30f..9b7497dc4a 100755 --- a/packaging/docker/dockerbuildi.sh +++ b/packaging/docker/dockerbuildi.sh @@ -5,7 +5,7 @@ set -e #set -x # dockerbuild.sh -# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 | loongarch64...] # -n [version number] # -p [password for docker hub] @@ -30,7 +30,7 @@ do passWord=$(echo $OPTARG) ;; h) - echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 | loongarch64...] " echo " -n [version number] " echo " -p [password for docker hub] " exit 0 diff --git a/packaging/release.sh b/packaging/release.sh index f250e667fa..c07331a0df 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -6,7 +6,7 @@ set -e #set -x # release.sh -v [cluster | edge] -# -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] +# -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64...] # -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] # -l [full | lite] @@ -19,7 +19,7 @@ set -e # set parameters by default value verMode=edge # [cluster, edge, cloud] verType=stable # [stable, beta] -cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] +cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 loongarch64...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] @@ -77,7 +77,7 @@ while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do ;; h) echo "Usage: $(basename $0) -v [cluster | edge] " - echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] " + echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64 ...] " echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " echo " -V [stable | beta] " echo " -l [full | lite] " @@ -216,7 +216,7 @@ else fi # check support cpu type -if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]]; then +if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]] || [[ "$cpuType" == "loongarch64" ]] ; then if [ "$verMode" == "edge" ]; then # community-version compile cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index e5ca9faacb..5e5d6656b4 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -902,9 +902,11 @@ void taosSetCoreDump(bool enable) { old_len = sizeof(old_usespid); +#ifndef __loongarch64 if (syscall(SYS__sysctl, &args) == -1) { // printf("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } +#endif // printf("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); @@ -918,9 +920,11 @@ void taosSetCoreDump(bool enable) { old_len = sizeof(old_usespid); +#ifndef __loongarch64 if (syscall(SYS__sysctl, &args) == -1) { // printf("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } +#endif // printf("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif @@ -989,4 +993,4 @@ bool taosCheckCurrentInDll() { #else return false; #endif -} \ No newline at end of file +} diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index bd662fa02c..fc9db5a3ff 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -19,7 +19,7 @@ */ #define _DEFAULT_SOURCE -#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) #include #endif @@ -512,7 +512,7 @@ static uint32_t table[16][256] = { 0x9c221d09, 0x6e2e10f7, 0x7dd67004, 0x8fda7dfa} }; -#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) static uint32_t long_shifts[4][256] = { {0x00000000, 0xe040e0ac, 0xc56db7a9, 0x252d5705, 0x8f3719a3, 0x6f77f90f, 0x4a5aae0a, 0xaa1a4ea6, 0x1b8245b7, 0xfbc2a51b, 0xdeeff21e, 0x3eaf12b2, 0x94b55c14, 0x74f5bcb8, 0x51d8ebbd, 0xb1980b11, 0x37048b6e, 0xd7446bc2, @@ -846,7 +846,7 @@ uint32_t crc32c_sf(uint32_t crci, crc_stream input, size_t length) { } return (uint32_t)crc ^ 0xffffffff; } -#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) /* Apply the zeros operator table to crc. */ static uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) { return shift_table[0][crc & 0xff] ^ shift_table[1][(crc >> 8) & 0xff] ^ shift_table[2][(crc >> 16) & 0xff] ^ @@ -857,7 +857,7 @@ static uint32_t shift_crc(uint32_t shift_table[][256], uint32_t crc) { version. Otherwise, use the software version. */ uint32_t (*crc32c)(uint32_t crci, crc_stream bytes, size_t len) = crc32c_sf; -#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) +#if !defined(_TD_ARM_) && !defined(_TD_MIPS_) && !defined(_TD_LOONGARCH_) /* Compute CRC-32C using the Intel hardware instruction. */ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { crc_stream next = buf; @@ -1012,7 +1012,7 @@ uint32_t crc32c_hw(uint32_t crc, crc_stream buf, size_t len) { #endif // #ifndef _TD_ARM_ void taosResolveCRC() { -#if defined _TD_ARM_ || defined _TD_MIPS_ || defined WINDOWS +#if defined _TD_ARM_ || defined _TD_MIPS_ || defined WINDOWS || defined _TD_LOONGARCH_ crc32c = crc32c_sf; #else int32_t sse42; diff --git a/tests/docs-examples-test/csharp.sh b/tests/docs-examples-test/csharp.sh index 8d1031ab8f..b193a3da41 100644 --- a/tests/docs-examples-test/csharp.sh +++ b/tests/docs-examples-test/csharp.sh @@ -28,10 +28,10 @@ taos -s "drop database if exists test" dotnet run --project optsJSON/optsJSON.csproj taos -s "create database if exists test" -dotnet run --project wsConnect/wsConnect.csproj -dotnet run --project wsInsert/wsInsert.csproj -dotnet run --project wsStmt/wsStmt.csproj -dotnet run --project wsQuery/wsQuery.csproj +# dotnet run --project wsConnect/wsConnect.csproj +# dotnet run --project wsInsert/wsInsert.csproj +# dotnet run --project wsStmt/wsStmt.csproj +# dotnet run --project wsQuery/wsQuery.csproj taos -s "drop database if exists test" taos -s "drop database if exists power" \ No newline at end of file From 010ac000b4b02e64de32a441e00b33b521915c74 Mon Sep 17 00:00:00 2001 From: huajingyun Date: Thu, 10 Nov 2022 22:44:58 +0800 Subject: [PATCH 09/46] Add loongarch64 architecture port. (#17982) Signed-off-by: Jingyun Hua Signed-off-by: Jingyun Hua From f55292dd6d91149f9141a04c54e3c6f870037773 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 11 Nov 2022 10:00:40 +0800 Subject: [PATCH 10/46] perf: optimize write --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 52c7a07c49..fed02b9d99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -568,7 +568,9 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i do { key.ts = row.pTSRow->ts; nRow++; - tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); + } code = tbDataDoPut(pMemTable, pTbData, pos, &row, 1); if (code) { goto _err; From df4fd528c012a7e36aff39d69d20333509fd8f7c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 11 Nov 2022 09:12:57 +0800 Subject: [PATCH 11/46] fix(stream): tbname max len to 192 --- source/dnode/vnode/src/tq/tqSink.c | 7 +++-- source/libs/executor/src/groupoperator.c | 15 ++++++----- source/libs/executor/src/scanoperator.c | 33 +++++++++++++++--------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 913fa67bd6..f782411084 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -244,7 +244,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem int32_t rows = pDataBlock->info.rows; - tqDebug("tq sink, convert block %d, rows: %d", i, rows); + tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); int32_t dataLen = 0; int32_t schemaLen = 0; @@ -486,7 +486,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d blkHead->uid = 0; blkHead->schemaLen = 0; - tqDebug("tq sink, convert block %d, rows: %d", i, rows); + tqDebug("tq sink, convert block2 %d, rows: %d", i, rows); int32_t dataLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -514,6 +514,9 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); } else { void* colData = colDataGetData(pColData, j); + if (k == 0) { + tqDebug("tq sink, row %d ts %" PRId64, j, *(int64_t*)colData); + } tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); } } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 47ee663abe..26a5f6838d 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -816,12 +816,12 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - setOperatorInfo(pOperator, "PartitionOperator", QUERY_NODE_PHYSICAL_PLAN_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "PartitionOperator", QUERY_NODE_PHYSICAL_PLAN_PARTITION, false, OP_NOT_OPENED, pInfo, + pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -900,7 +900,7 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { // TODO check tbname validity if (pData != (void*)-1) { memset(pDest->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(pDest->info.parTbName, varDataVal(pData), len); /*pDest->info.parTbName[len + 1] = 0;*/ } else { @@ -1099,11 +1099,12 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols); - setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, - destroyStreamPartitionOperatorInfo, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b83fac9fa6..945753cc53 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -946,7 +946,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; - setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, + pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); @@ -980,7 +981,8 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pInfo->dataReader = pReadHandle; // pInfo->prevGroupId = -1; - setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); return pOperator; } @@ -1136,8 +1138,10 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi goto _error; } - setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); return pOperator; _error: @@ -1581,7 +1585,7 @@ static void calBlockTbName(SExprSupp* pTbNameCalSup, SSDataBlock* pBlock) { // TODO check tbname validation if (pData != (void*)-1 && pData != NULL) { memset(pBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN); + int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); memcpy(pBlock->info.parTbName, varDataVal(pData), len); /*pBlock->info.parTbName[len + 1] = 0;*/ } else { @@ -2351,7 +2355,8 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT pInfo->vnode = pHandle->vnode; pInfo->sContext = pHandle->sContext; - setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, + pTaskInfo); pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); return pOperator; @@ -2537,7 +2542,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->partitionSup.needCalc = false; - setOperatorInfo(pOperator, "StreamScanOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "StreamScanOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo, + pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; @@ -4175,7 +4181,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); return pOperator; @@ -4305,7 +4312,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; - setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, + pTaskInfo); initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -4815,11 +4823,12 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, - destroyTableMergeScanOperatorInfo, getTableMergeScanExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, + getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; From 13680e2ee6ac6594e496695176c452edd0391e16 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 10:25:16 +0800 Subject: [PATCH 12/46] chore: set stream fill history on for tsma --- source/dnode/mnode/impl/src/mndSma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 56e725fac7..698c07d9bc 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -532,6 +532,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.sql = strdup(pCreate->sql); streamObj.smaId = smaObj.uid; streamObj.watermark = pCreate->watermark; + streamObj.fillHistory = STREAM_FILL_HISTORY_ON; streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE; streamObj.triggerParam = pCreate->maxDelay; streamObj.ast = strdup(smaObj.ast); From 4a8196b5205a5805cea4576476a74adc42e3feec Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Fri, 11 Nov 2022 10:27:54 +0800 Subject: [PATCH 13/46] fix: update stb stats cache when creating/dropping child tables --- source/dnode/vnode/src/inc/meta.h | 1 + source/dnode/vnode/src/meta/metaQuery.c | 10 ++++++++++ source/dnode/vnode/src/meta/metaTable.c | 8 +++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 9e2fe4aaf0..2ceae91f7c 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -70,6 +70,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo); int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); +void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta); struct SMeta { TdThreadRwlock lock; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 4aabd39800..620022c06d 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1445,3 +1445,13 @@ int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo) { _exit: return code; } + +void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta) { + SMetaStbStats stats = {0}; + + if (metaStatsCacheGet(pMeta, uid, &stats) == TSDB_CODE_SUCCESS) { + stats.ctbNum += delta; + + metaStatsCacheUpsert(pMeta, &stats); + } +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 6dadce80ca..5921adfbfa 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -371,7 +371,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // update uid index metaUpdateUidIdx(pMeta, &nStbEntry); - metaStatsCacheDrop(pMeta, nStbEntry.uid); + // metaStatsCacheDrop(pMeta, nStbEntry.uid); metaULock(pMeta); @@ -450,6 +450,10 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe #endif ++pMeta->pVnode->config.vndStats.numOfCTables; + + metaWLock(pMeta); + metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1); + metaULock(pMeta); } else { me.ntbEntry.ctime = pReq->ctime; me.ntbEntry.ttlDays = pReq->ttl; @@ -670,6 +674,8 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn); --pMeta->pVnode->config.vndStats.numOfCTables; + + metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) From 5651bde6fe6bbe7f474634d6267e40d551d4e972 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 10:54:46 +0800 Subject: [PATCH 14/46] enh: use lock in buf pool only for rsma vnode --- source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/vnd/vnodeBufPool.c | 30 +++++++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 8f8691cfc2..ecb34cd9b4 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -65,6 +65,7 @@ struct SVBufPool { SVnode* pVnode; volatile int32_t nRef; TdThreadSpinlock lock; + bool isLock; int64_t size; uint8_t* ptr; SVBufPoolNode* pTail; diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 6ac2ce1c16..86cd8af4d6 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -27,10 +27,15 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) return -1; } - if (taosThreadSpinInit(&pPool->lock, 0) != 0) { - taosMemoryFree(pPool); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + if (VND_IS_RSMA(pVnode)) { + if (taosThreadSpinInit(&pPool->lock, 0) != 0) { + taosMemoryFree(pPool); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPool->isLock = true; + } else { + pPool->isLock = false; } pPool->next = NULL; @@ -49,7 +54,9 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) static int vnodeBufPoolDestroy(SVBufPool *pPool) { vnodeBufPoolReset(pPool); - taosThreadSpinDestroy(&pPool->lock); + if (pPool->isLock) { + taosThreadSpinDestroy(&pPool->lock); + } taosMemoryFree(pPool); return 0; } @@ -114,7 +121,10 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { void *p = NULL; ASSERT(pPool != NULL); - taosThreadSpinLock(&pPool->lock); + if (pPool->isLock) { + taosThreadSpinLock(&pPool->lock); + } + if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node p = pPool->ptr; @@ -125,7 +135,9 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pNode = taosMemoryMalloc(sizeof(*pNode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - taosThreadSpinUnlock(&pPool->lock); + if (pPool->isLock) { + taosThreadSpinUnlock(&pPool->lock); + } return NULL; } @@ -138,7 +150,9 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pPool->size = pPool->size + sizeof(*pNode) + size; } - taosThreadSpinUnlock(&pPool->lock); + if (pPool->isLock) { + taosThreadSpinUnlock(&pPool->lock); + } return p; } From 936daff09bd38716b0c9f87a0ad5966b2bbe47e9 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 10:57:48 +0800 Subject: [PATCH 15/46] chore: revert the code change --- source/dnode/vnode/src/inc/vnd.h | 1 - source/dnode/vnode/src/vnd/vnodeBufPool.c | 30 +++++++---------------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index ecb34cd9b4..8f8691cfc2 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -65,7 +65,6 @@ struct SVBufPool { SVnode* pVnode; volatile int32_t nRef; TdThreadSpinlock lock; - bool isLock; int64_t size; uint8_t* ptr; SVBufPoolNode* pTail; diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 86cd8af4d6..3069e5224e 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -27,15 +27,10 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) return -1; } - if (VND_IS_RSMA(pVnode)) { - if (taosThreadSpinInit(&pPool->lock, 0) != 0) { - taosMemoryFree(pPool); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pPool->isLock = true; - } else { - pPool->isLock = false; + if (taosThreadSpinInit(&pPool->lock, 0) != 0) { + taosMemoryFree(pPool); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } pPool->next = NULL; @@ -54,9 +49,7 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) static int vnodeBufPoolDestroy(SVBufPool *pPool) { vnodeBufPoolReset(pPool); - if (pPool->isLock) { - taosThreadSpinDestroy(&pPool->lock); - } + taosThreadSpinDestroy(&pPool->lock); taosMemoryFree(pPool); return 0; } @@ -121,9 +114,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { void *p = NULL; ASSERT(pPool != NULL); - if (pPool->isLock) { - taosThreadSpinLock(&pPool->lock); - } + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node @@ -135,9 +126,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pNode = taosMemoryMalloc(sizeof(*pNode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - if (pPool->isLock) { - taosThreadSpinUnlock(&pPool->lock); - } + taosThreadSpinUnlock(&pPool->lock); return NULL; } @@ -150,9 +139,8 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pPool->size = pPool->size + sizeof(*pNode) + size; } - if (pPool->isLock) { - taosThreadSpinUnlock(&pPool->lock); - } + + taosThreadSpinUnlock(&pPool->lock); return p; } From 37b7e34fe26e26e97de3a0487065ebe4f103fbd3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 10:58:25 +0800 Subject: [PATCH 16/46] chore: revert the code change --- source/dnode/vnode/src/vnd/vnodeBufPool.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 3069e5224e..6ac2ce1c16 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -115,7 +115,6 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { ASSERT(pPool != NULL); taosThreadSpinLock(&pPool->lock); - if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node p = pPool->ptr; @@ -139,7 +138,6 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pPool->size = pPool->size + sizeof(*pNode) + size; } - taosThreadSpinUnlock(&pPool->lock); return p; } From 35b79e7e6989a02e6dcc7e52a39798dd1ec62088 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 11 Nov 2022 11:24:24 +0800 Subject: [PATCH 17/46] fix: the problem of null in case when expression --- source/libs/parser/src/parTranslater.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7897623ab1..0e5cb14208 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1878,6 +1878,7 @@ static int32_t rewriteIsTrue(SNode* pSrc, SNode** pIsTrue) { static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseWhen) { bool first = true; + bool allNullThen = true; SNode* pNode = NULL; FOREACH(pNode, pCaseWhen->pWhenThenList) { SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; @@ -1889,12 +1890,28 @@ static EDealRes translateCaseWhen(STranslateContext* pCxt, SCaseWhenNode* pCaseW } pWhenThen->pWhen = pIsTrue; } - if (first || dataTypeComp(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType) < 0) { - pCaseWhen->node.resType = ((SExprNode*)pNode)->resType; + + SExprNode* pThenExpr = (SExprNode*)pNode; + if (TSDB_DATA_TYPE_NULL == pThenExpr->resType.type) { + continue; + } + allNullThen = false; + if (first || dataTypeComp(&pCaseWhen->node.resType, &pThenExpr->resType) < 0) { + pCaseWhen->node.resType = pThenExpr->resType; } first = false; } + if (allNullThen) { + if (NULL != pCaseWhen->pElse) { + pCaseWhen->node.resType = ((SExprNode*)pCaseWhen->pElse)->resType; + } else { + pCaseWhen->node.resType.type = TSDB_DATA_TYPE_NULL; + pCaseWhen->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; + return DEAL_RES_CONTINUE; + } + } + FOREACH(pNode, pCaseWhen->pWhenThenList) { SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; if (!dataTypeEqual(&pCaseWhen->node.resType, &((SExprNode*)pNode)->resType)) { From 07ed17806540e8bc1c51006419f3f708e3cb581e Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Fri, 11 Nov 2022 11:54:52 +0800 Subject: [PATCH 18/46] test: add case for tmq in replica3 --- tests/parallel_test/cases.task | 1 + tests/system-test/7-tmq/tmqSubscribeStb-r3.py | 30 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6775760ee4..8955a65dfa 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -714,6 +714,7 @@ ,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py +,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 ,,,system-test,python3 ./test.py -f 2-query/between.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/distinct.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/varchar.py -Q 2 diff --git a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py index 01f1ca5b15..85222a941b 100644 --- a/tests/system-test/7-tmq/tmqSubscribeStb-r3.py +++ b/tests/system-test/7-tmq/tmqSubscribeStb-r3.py @@ -1,26 +1,27 @@ -from distutils.log import error -import taos -import sys -import time -import socket import os -import threading -import subprocess import platform +import socket +import subprocess +import sys +import threading +import time +from distutils.log import error -from util.log import * -from util.sql import * +import taos from util.cases import * -from util.dnodes import * -from util.dnodes import TDDnodes -from util.dnodes import TDDnode from util.cluster import * from util.common import * +from util.dnodes import * +from util.dnodes import TDDnode, TDDnodes +from util.log import * +from util.sql import * + sys.path.append("./6-cluster") sys.path.append("./7-tmq") -from tmqCommon import * -from clusterCommonCreate import * from clusterCommonCheck import clusterComCheck +from clusterCommonCreate import * +from tmqCommon import * + class TDTestCase: def __init__(self): @@ -265,6 +266,7 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("================= restart dnode 2===========================") cluster.dnodes[1].stoptaosd() cluster.dnodes[1].starttaosd() From 5f97ea3ce1d8e48f4fbcc319693539745a602d65 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 11 Nov 2022 13:11:48 +0800 Subject: [PATCH 19/46] perf: more optimize --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index fed02b9d99..654afe1b6a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -458,11 +458,10 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p } static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { - int8_t level = 1; - int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); - const uint32_t factor = 4; + int8_t level = 1; + int8_t tlevel = TMIN(pSl->maxLevel, pSl->level + 1); - while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) { + while ((taosRandR(&pSl->seed) & 0x3) == 0 && level < tlevel) { level++; } From 11d7fc6042b39dbe530494479fdaa3809f9e9ec0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 11 Nov 2022 13:29:35 +0800 Subject: [PATCH 20/46] fix: taosbenchmark ver on windows (#18039) * feat: taosbenchmark support query fails then continue * fix: taosbenchmark version for windows --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index d18d85171d..82a7052125 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG a921bd4 + GIT_TAG 23e2b73 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From d9faec5e9f48cd8593ca8304e0d334df8120c38e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 13:53:16 +0800 Subject: [PATCH 21/46] enh: lock buf pool only for rsma vnode --- source/dnode/vnode/src/inc/vnd.h | 16 ++++++++-------- source/dnode/vnode/src/vnd/vnodeBufPool.c | 20 ++++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 8f8691cfc2..29af2bda67 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -61,14 +61,14 @@ struct SVBufPoolNode { }; struct SVBufPool { - SVBufPool* next; - SVnode* pVnode; - volatile int32_t nRef; - TdThreadSpinlock lock; - int64_t size; - uint8_t* ptr; - SVBufPoolNode* pTail; - SVBufPoolNode node; + SVBufPool* next; + SVnode* pVnode; + TdThreadSpinlock* lock; + volatile int32_t nRef; + int64_t size; + uint8_t* ptr; + SVBufPoolNode* pTail; + SVBufPoolNode node; }; int32_t vnodeOpenBufPool(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 6ac2ce1c16..49f9cfd76d 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -27,10 +27,14 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) return -1; } - if (taosThreadSpinInit(&pPool->lock, 0) != 0) { - taosMemoryFree(pPool); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + if (VND_IS_RSMA(pVnode)) { + if (taosThreadSpinInit(pPool->lock, 0) != 0) { + taosMemoryFree(pPool); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } else { + pPool->lock = NULL; } pPool->next = NULL; @@ -49,7 +53,7 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) static int vnodeBufPoolDestroy(SVBufPool *pPool) { vnodeBufPoolReset(pPool); - taosThreadSpinDestroy(&pPool->lock); + if (pPool->lock) taosThreadSpinDestroy(pPool->lock); taosMemoryFree(pPool); return 0; } @@ -114,7 +118,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { void *p = NULL; ASSERT(pPool != NULL); - taosThreadSpinLock(&pPool->lock); + if(pPool->lock) taosThreadSpinLock(pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node p = pPool->ptr; @@ -125,7 +129,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pNode = taosMemoryMalloc(sizeof(*pNode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - taosThreadSpinUnlock(&pPool->lock); + if (pPool->lock) taosThreadSpinUnlock(pPool->lock); return NULL; } @@ -138,7 +142,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pPool->size = pPool->size + sizeof(*pNode) + size; } - taosThreadSpinUnlock(&pPool->lock); + if(pPool->lock) taosThreadSpinUnlock(pPool->lock); return p; } From 057af11ae89122fb5eee2ae534f17ff76acd8ffc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 11 Nov 2022 13:57:38 +0800 Subject: [PATCH 22/46] format more --- source/dnode/vnode/src/vnd/vnodeBufPool.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 49f9cfd76d..2fa88badf6 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -118,7 +118,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { void *p = NULL; ASSERT(pPool != NULL); - if(pPool->lock) taosThreadSpinLock(pPool->lock); + if (pPool->lock) taosThreadSpinLock(pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node p = pPool->ptr; @@ -142,7 +142,7 @@ void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { pPool->size = pPool->size + sizeof(*pNode) + size; } - if(pPool->lock) taosThreadSpinUnlock(pPool->lock); + if (pPool->lock) taosThreadSpinUnlock(pPool->lock); return p; } From dd4333b14058cc378d0070a5c21659defdd2f64a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 11 Nov 2022 14:11:42 +0800 Subject: [PATCH 23/46] enh: optimize the error message when illegally modifying the column length --- source/libs/parser/src/parUtil.c | 2 +- source/util/src/terror.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 00a72a1946..e8c3f2fa8d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -138,7 +138,7 @@ static char* getSyntaxErrFormat(int32_t errCode) { case TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY: return "Primary timestamp column cannot be dropped"; case TSDB_CODE_PAR_INVALID_MODIFY_COL: - return "Only binary/nchar column length could be modified"; + return "Only binary/nchar column length could be modified, and the length can only be increased, not decreased"; case TSDB_CODE_PAR_INVALID_TBNAME: return "Invalid tbname pseudo column"; case TSDB_CODE_PAR_INVALID_FUNCTION_NAME: diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0e6568d692..a1162d2e94 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -521,7 +521,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TIMELINE_FUNC, "Invalid timeline fu TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PASSWD, "Invalid password") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Invalid alter table statement") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY, "Primary timestamp column cannot be dropped") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar column length could be modified") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_MODIFY_COL, "Only binary/nchar column length could be modified, and the length can only be increased, not decreased") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TBNAME, "Invalid tbname pseudo column") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_FUNCTION_NAME, "Invalid function name") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COMMENT_TOO_LONG, "Comment too long") From 2386e0b513ea6153eb5da14d526677a73263cf0c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 11 Nov 2022 14:37:31 +0800 Subject: [PATCH 24/46] refactor(tmq): add more strict drop condition check --- source/dnode/mnode/impl/src/mndTopic.c | 37 +++++++++++++++++++------- source/dnode/vnode/src/tq/tq.c | 6 ++--- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e1ca1d2708..522036afa2 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -637,6 +637,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (pIter == NULL) break; if (pConsumer->status == MQ_CONSUMER_STATUS__LOST_REBD) continue; + int32_t sz = taosArrayGetSize(pConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { char *name = taosArrayGetP(pConsumer->assignedTopics, i); @@ -649,6 +650,33 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } } + + sz = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < sz; i++) { + char *name = taosArrayGetP(pConsumer->rebNewTopics, i); + if (strcmp(name, pTopic->name) == 0) { + mndReleaseConsumer(pMnode, pConsumer); + mndReleaseTopic(pMnode, pTopic); + terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; + mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", + dropReq.name, pConsumer->consumerId, pConsumer->cgroup); + return -1; + } + } + + sz = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < sz; i++) { + char *name = taosArrayGetP(pConsumer->rebRemovedTopics, i); + if (strcmp(name, pTopic->name) == 0) { + mndReleaseConsumer(pMnode, pConsumer); + mndReleaseTopic(pMnode, pTopic); + terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; + mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb remove)", + dropReq.name, pConsumer->consumerId, pConsumer->cgroup); + return -1; + } + } + sdbRelease(pSdb, pConsumer); } @@ -675,15 +703,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); -#if 0 - if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { - ASSERT(0); - mndTransDrop(pTrans); - mndReleaseTopic(pMnode, pTopic); - return -1; - } -#endif - // TODO check if rebalancing if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { /*ASSERT(0);*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9c377fe7f5..f75dce8231 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -582,10 +582,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = -1; } - tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, send data blockNum:%d, offset type:%d, uid:%" PRId64 - ", version:%" PRId64 "", + tqDebug("tmq poll: consumer %" PRId64 + ", subkey %s, vg %d, send data blockNum:%d, offset type:%d, uid/version:%" PRId64 ", ts:%" PRId64 "", consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type, - dataRsp.rspOffset.uid, dataRsp.rspOffset.version); + dataRsp.rspOffset.uid, dataRsp.rspOffset.ts); tDeleteSMqDataRsp(&dataRsp); return code; From 603cc02671df200b81d142ecb8f6ed1dbcdb5dc4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 11 Nov 2022 14:37:31 +0800 Subject: [PATCH 25/46] refactor(tmq): add more strict drop condition check --- include/common/tmsg.h | 21 +++++++++------ source/dnode/mnode/impl/src/mndTopic.c | 37 +++++++++++++++++++------- source/dnode/vnode/src/tq/tq.c | 6 ++--- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b16b5a2d4b..99c5c72e2f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -418,13 +418,17 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { buf = taosDecodeVariantI32(buf, &pSW->nCols); buf = taosDecodeVariantI32(buf, &pSW->version); - pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); - if (pSW->pSchema == NULL) { - return NULL; - } + if (pSW->nCols > 0) { + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + return NULL; + } - for (int32_t i = 0; i < pSW->nCols; i++) { - buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); + for (int32_t i = 0; i < pSW->nCols; i++) { + buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); + } + } else { + pSW->pSchema = NULL; } return (void*)buf; } @@ -839,7 +843,7 @@ typedef struct { int64_t dbId; int32_t vgVersion; int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT - int64_t stateTs; // ms + int64_t stateTs; // ms } SUseDbReq; int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); @@ -2990,7 +2994,8 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE } static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - if (pSubTopicEp->schema.nCols) taosMemoryFreeClear(pSubTopicEp->schema.pSchema); + taosMemoryFreeClear(pSubTopicEp->schema.pSchema); + pSubTopicEp->schema.nCols = 0; taosArrayDestroy(pSubTopicEp->vgs); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e1ca1d2708..522036afa2 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -637,6 +637,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { if (pIter == NULL) break; if (pConsumer->status == MQ_CONSUMER_STATUS__LOST_REBD) continue; + int32_t sz = taosArrayGetSize(pConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { char *name = taosArrayGetP(pConsumer->assignedTopics, i); @@ -649,6 +650,33 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return -1; } } + + sz = taosArrayGetSize(pConsumer->rebNewTopics); + for (int32_t i = 0; i < sz; i++) { + char *name = taosArrayGetP(pConsumer->rebNewTopics, i); + if (strcmp(name, pTopic->name) == 0) { + mndReleaseConsumer(pMnode, pConsumer); + mndReleaseTopic(pMnode, pTopic); + terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; + mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", + dropReq.name, pConsumer->consumerId, pConsumer->cgroup); + return -1; + } + } + + sz = taosArrayGetSize(pConsumer->rebRemovedTopics); + for (int32_t i = 0; i < sz; i++) { + char *name = taosArrayGetP(pConsumer->rebRemovedTopics, i); + if (strcmp(name, pTopic->name) == 0) { + mndReleaseConsumer(pMnode, pConsumer); + mndReleaseTopic(pMnode, pTopic); + terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; + mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb remove)", + dropReq.name, pConsumer->consumerId, pConsumer->cgroup); + return -1; + } + } + sdbRelease(pSdb, pConsumer); } @@ -675,15 +703,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mInfo("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); -#if 0 - if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { - ASSERT(0); - mndTransDrop(pTrans); - mndReleaseTopic(pMnode, pTopic); - return -1; - } -#endif - // TODO check if rebalancing if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) { /*ASSERT(0);*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9c377fe7f5..f75dce8231 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -582,10 +582,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { code = -1; } - tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, send data blockNum:%d, offset type:%d, uid:%" PRId64 - ", version:%" PRId64 "", + tqDebug("tmq poll: consumer %" PRId64 + ", subkey %s, vg %d, send data blockNum:%d, offset type:%d, uid/version:%" PRId64 ", ts:%" PRId64 "", consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.blockNum, dataRsp.rspOffset.type, - dataRsp.rspOffset.uid, dataRsp.rspOffset.version); + dataRsp.rspOffset.uid, dataRsp.rspOffset.ts); tDeleteSMqDataRsp(&dataRsp); return code; From 504a9fec3150c8b66f93ca9a37b962d1ec1dfe84 Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Fri, 11 Nov 2022 15:00:43 +0800 Subject: [PATCH 26/46] docs: add WeChat QR code of xiao-t --- docs/zh/05-get-started/index.md | 8 +++++++- docs/zh/05-get-started/tdengine.webp | Bin 0 -> 24678 bytes 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/zh/05-get-started/tdengine.webp diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 20f8235d87..cbdc332477 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -12,4 +12,10 @@ import DocCardList from '@theme/DocCardList'; import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; -``` \ No newline at end of file +``` + +### 开发者技术交流群 + +微信扫描下面二维码,加“小 T”为好友,即可加入“物联网大数据技术前沿群”,与大家共同交流物联网大数据技术应用、TDengine 使用问题和技巧等话题。 + + diff --git a/docs/zh/05-get-started/tdengine.webp b/docs/zh/05-get-started/tdengine.webp new file mode 100644 index 0000000000000000000000000000000000000000..e1bc0a75acd9f552ee2f5ceb3877670bfb49703b GIT binary patch literal 24678 zcmb4q(~>YutnJvgZO?pT+qP}nwr$(CZQHhO?th;bIJGOCo9z!|_wSfG|Eb)=60?}0&SW>f4n?Tt0bYCBJ8%CitR*g&yUg?ShrIV-NbfAK^e^nM z37`8n<)!KV>5Kjy{s9k6*Yc(Bo9#XCjxUmZ=~M54d+y`p)$Y#k@UQMK?vLfS?t%VI z@4yea&&==C>*+h}neT~5z(?W7>&xzP?QQQl@5=Z0AbLMe=hyD1@0;)H?6q!*3q$624LIv5(xZ-6y~g=f9Y;ueU3^*Sx-zlGbJ&#}Mq#a;md(nm-@`gn0j??8U3V#C+B}2ByP{I+$4X#79#(`K9*^{%csK=?d{LmDRXqv2 zbPesF6+(dgrRFOB-28V=!fPp7U!qP&=-)y1WqfcW`DBR^F4ulkqOKxc%qUUCDP2YIxbERk zRO-%?!`vGl(wif&(nl@O-pa5Qv~VLuk889$RzRXknT??e$S((?9lRsa_KMK9VA7so zjloGRW*jqAK`zI1%13EtQwQQr3!^}?0*+-q{8D8Bx&I38x>T}S@J`>fd}D^`-Z=4Z zXbF7KZ3eAz@eh8wxmB|03ki`NkMbw_Y7_E70=MLpG@txehQ{nkF!JOa#p%|7=tHjo z3)>kk$$DX0p%9$F6ocedg)>Rxi$Lq8$mfMR9R+FvtFS{$pdLZED0&6Jf=M~Sh&6Cq zwlJ)q$<9Hkj1~-^x<=)Tv*h2KuK}rP;T2~=gHRd*GYx7L;;Nr25m;l-!on!e`?S>t zP5pm`>gOtk$OYFQRn?6@jOL;Gnwt@i>C`o1{UxJB{KG`1M@6`MjQma_iTam4!lulN z-Sfn6V)}MwavI5eyg2l)&zS7U#{T;h*j&X+j44Z6a*`GKZdYXEax;EEac1dCG((dB%TCdaf2Scov1p9W?M+oxvlU@H|}Zy52`!#TKx4hnE{IUxDgW&p=$i9UlVTz?vF9)5y?f?I{cVL zK$E>;4l762sEn$hQ^nV{Gm(i3c*3gVH+d=$5)f*Q&qUTED*T|HAyv)AV1#1|g-mtC z$t$$}Ft+`gc(P*6`)t5U0Do^i`_@2dc{VW~D&znGqm?dm4w37GdQ z9LphAAL$UAL&Y>^GI7|+?t~vfcz+EQcE^UQ+Y2aI=dLu>KfkK2$DIHjrWW>sbQq;{ zBsnIrDzC}lAHk9f~tl23m zKS_C(>g<}@y!jJMI$lU)gQw#CA{@2+_K99@!}1;w~CYJ~CWX2&&f!4l`KwE(V+vXL>Q&WPO6gRQw%UN#u> zBTx6oaci~PUc1iYEb%@Hrj9c|^$VPYux3Z6*6btdU^+@sXZgs;aVP?Vxzcg&83~Gc zhHg9)#P2c+2om0C(&}~-zLh0|9u@~gOOu$zuf1?dF2_!rH#x__%q`gWR{Yh|+@Zzsl8NB+pK`FS=D%hQ(&jH6 z>%?D=`a{5Orp{1^z34Ogcy{MtL%~KHa0@(>kL9weO3zC$^n~oHo9A{ES`qd4BsOYn zyZUr*lBr;^^fm$%bh8HJowPEJP*>pn@Het7h)abBb7Pn&Cj6o?Yv6ux9oM7n)KYQ= zC73Td(`0u|-&!17DDu z->5uEhkWG4I^43k57L`qUZY#}efV4neZ+epCiz`>8u)V|iaOd18bIog!Rz*(?muna z(}WRwbe~uvvPP_F&WiS?L7og_9-;gHeVNHx-XELnqF{0^=l5i@ik!=xTE;l)%8_(U zv1+p&SbvV-rHp^7@MTm=%YxSsJoaLP4Cs|MDpb516*KaV=_^3P^#fRCO(k#9ak z|Cp_poexN1uYAm$!-KHPv46LTSm3y(6sm-E(IC;avqdq8HOf%*Lh^jkM!ru(FY}t} zmi1@h)1m^i1Pr=E2FE`2JoKmHLyqPF=s=s4$~JUqz7|J$0M0HqW8O1_5H!eurgGpF zyA@5r@zW)EdwJqbsO}D|o86y4=a5KVNHJe}LOw_{oZOC1VAw_)s-aK)k8p@MbTG5_ z(3~{G-ZH8Ay2SmJefC*h=kAG7O^_Z__8Q5B|9{`-TJ>hJ7VZcj1zkuzXC>29SXHfr9V}&Y=opQ|?UBl@IJ(gvu(VmK(y25~vv}d1L?<(p-4flk zMwG^*<-1bc>~ucE?ph>_V;$r(@4pA5ZG=+)BW_D}3b1W2nr*Or`8t1&--QiIsu1;& zzi3INFf_R2dU3A6xN-@ksG7s{D@oZ{l4aE#`N|5Vh4**+hzuDvj9u450kxRl5H)t4ZOYtT~xZjE&@2rpzK4&FLg*dE__P*cx-ndXAN!8he;ZVITil^S? z9suzV(Fh}B2dxPvCKK_trO}j|AAP)-_(i3fN z6}~t(gAAM*d(lKSqgy)n&MuXaD-<8a=LZpap9xJu~1ZD!VRVnT7=PZKe zm!&I?eL*Gm4Nym~iK)eF<3|wp$$}B#d!P_}-pF!@7@3v(p0+O*_z5JqT{BK>HX9{F z#QxQzN`7`vr7mLkN?$%%Il6N|pkeGC?Dq`LB-G@~!jgs=I8e4#+rkWGL=5u3Xg4Zi z52{O54hX`~_p52cxebkn-P(H#Fr0wVzg(i;F9j0xJcfQvb3}l}RLQ=ki0H7X%W3;t zZ#xKv(+9(}aIBxEEvpOdKdjas$+$!Q`t~7~t#z*71`$iLyViSnGoH=a7cVlQ`3(N} zP4@II z-hB#0+Egk^AElUiGph{Qo0^zzT8bH^YerLIj8N0BXH@_gN+Oppn^|GWs^+Ec3}yht z;3re#2$I0%s_$OL2UO;<2Tb$|u($41bn>V-PW$w8lZ*}Wgq6wM(fCo$wN9_c`LpWHQ^Ito_qDcy<(jCU~{xW ziaKCpBf4_$LX_}*UA(WQln7{R4>WB*EflS`e7`DzIY8atBm;6JL6n5QT=AJ_%;0$D z$<^y*D7fu#UCUbke`YoRCq4w<5Agf9gZ3X61pww`@Igz?rh4QKDe$!y^R5Z3ismrl zz%PST+w6Xl`3i zu#SE{it%NCVQSKZ-wMy%^qzM*EnDf{8b00{xnlZ`M+)9>g5eEsU_r!P5>Ex8;$3WpMe7T=b*BJmz zht(C(LURgFb06BHZ*7Z$hm&k+WO=P(j!|e4_f{u&7_}Q%D#X;zf z8dI`1QW=RWFGlOpH|q_NI`A-)n{JsrJe9G9^8G4Ooo6CpC3h1Ae!F)Q@Y;DZhEu;l z+JhreR8S^Fz4gUSwHC}MM51U*iiWW8{|Y}>yq-fG0MrCtAoCWBoLyujZYWdFcr>IJ zxlT}?r9DO$T_*r3wqnxynk#r5Puer1|D)}CJ?2bge|sk#&~fzy$4Oi@q2zg?y>aGE z{f64GG<1$Dp4JRepr!d+B(9&GF%I{Yhp2MJK6fPU4!+xTH*dU(K65UU$Mj$|dJQV& zbi_OvEX3?+k*?X4TUEKdrM)URYvNVX2&RcjQMSZS@vRMA3VK`Rnj}Q(K6U0aF$e_S z0z1FE?~Ioi{h6K=DDMo?;uL}bDfD+bIU{a)4NvjR?-0xC-!yXYY{T-QPpWb&6$GyZ zFPi8i7m!#ie?ia`oGc#X?(ooMPOlo!aD&q2*&_ z?Wc&gj#PT!1!Xu__!7Y>yl9fzZsakl3XH@l-YK@Vxo=xcW8u2$pM@qyCXKdy zD2FETPhMD0m~E6}L??%m6emMmKUq5#bAU)PLpdla|-rO;6c(lz4uS}#^U@Lk#3EjZU_7{Q82!-y@kzyOzQP!z{g z<0ZtQ?WdO%xj(Vj~S@b`khd;(~53pbPHi~ z!|av-c0%YM0t}F(lB+bA&j{?i2o3|h9d5~y7J7;*iZ4M*B&^zR2r@05Ji5+(UC9@$= zig7I;?MhyTR&Di>DGyHxCx;9{H5i(Kl^ZupNjj75V+Ay*W;@*GnKJZe&0-)VkXAA< zquE5SJu4n5)E=c9ci5Oy;wWw<=7*kq-};ihZ2ON}{=8q~gH8?J)gVrlS(w#GIs8j< z@H2WSKMTb%AAf$@mtfU5K%#RhD(YrL%z4}=yRH9*o?9wA1$ZKE@xX&aY@2Yv2#-fw z260Au>P@4XCMl3plF|JD@ukpMexp|KG-b&vHtKM$NaKhOv`MQ?&C?9-Z(a|*bG>y7 z>W?yIYboIdYBrfnYxmD3Xb9CXZQWN8bbSkC=ywQUp0ADTRG7H&NT6_S!4F< zP}W}K!6QhNn-ut;@u7^x&h_t|9pQ>XvU~?BC|9e~TqGs{)z3YAD?=Pdk z@DP%6@C$W@o@`5JG$3uFPC&)rd$G}^vYt%OJI*zr1UQIEafv*GFNjKObOF-NY!H(w zcygza$qp%7yvg9(4$4ui@?gTQ=x%n5Gfn}^auHcM0Wy=2o3LrRMUW4oKzPN-);au1w)=$Rtkx_ck!{qx$Ggmi zXWl@DqK{pl$XQ@xHGftSMD)H)!ECGmM76ry?!vRik>-H*Lsh24oYu>j)i!_!{^JW& zXy%@~3#2K?swsnEq~T&WAK@!TuprsXSy=K<&kcdO^> zJC3-!!{ZcqM(6v4!)7lcG-kur9f(`)BMbvWy>LhQ?bPKNoE8&vccP*Fm~bWx`O|TB zE}I=Ol?WJa3Vsp?S9#L)H&Y9Kz5+;0M|BSNdY8JWYWD@K=LV4ObR`8uAtd#oz~7A> z@wbSbAp#h|HyT=yK&z5X$jiJLM$O6viANDu9f4IHN=szTW87eg>MOGHJF~|xXPK;U z@qDWR)+;Re9*6Hclr`XG674-u!3SLwB>(DCGiSn;4#U9k+C^;`3&tL&Fy$R~ zR0WSP*sDdQsS9ZxIGo<;Iww{tnD^EkBh zij9m}aqc=R2GEr$%bY6WA?+#~&v5%G995(P^JlW1+~4^KqZpvH&ICFrQ-?0-QT=&y z1^LfR<2Q=cs_^{R;{#JUk~{slVsLx*{`SicZa<1^@+EM4yD<#3&_5sSa7e%9(C@Uh z$mv!7+qAib*(t{J77oX;EEzH(Yz6t>-TNS%i*hH~Ubxs0-Rz6>SE1FE$#2WH(F?~*?ilBzyL z^BCcom2hDdd~NGKgf~qhHot4gnjo>F9CljI>Aw@A%gx%u*-e^ z9^u3)5Mg`)2EMyt-l^7|=gzO5`8)#dTJ!CHv#nhU4JVnPFd9wjVC4gFBBbz#VNNWa zuuui=MugFNW70G_O%pk=E5W1mxG&b7XIS8M^-R(RS)Wx@lwDJUaxAE$=|9nIs$A7qw5+RY;|_{wBr@!V#r}k-aO1(4 z)V;_M>$x@snG!7h+1wU z1=iI3+d*us@j(UyKrMKlo3WO}dt*se72qHiTxz5yUvnQ7v%+*ifs_@uRl$LD1gf=?xn7vehJIM`fWG9Mh&|ZgGl+QPqdO`8yt}o$@wSj3D32 z^1xDq<7OOPXPFZ;A(yl3J+P+NyGPg5W%Hw>s8G@>L)L2l22Xh!Q zjkC+0xnaz`HK80+p}3MbF%)CVtf|v)&QzPS{Fgyb?Y+(7^^Szc!id6a1Px~srsMEo zEaAE5PN3bQYOBM0fSOghN(NQh)HP89e?24iG-%%qKt*SqxL!U}^l-7RorF@X(iuIh$2)xM_Dh9{FqTq#(7mypQbWz#(8}H!eb8F!6%A z9yRgnL#6MrsGdElmqMVaj%8oUq<>^-Y|SM?MfrFZnL5Z*8M_&uQFOn@)4m_Y$%A9# z#k7_07`6Ivp`Ban<)|8dQRxA?iBu+?l8$Yu&@zs|Fi=cOLrTHCfod}f+4Hc#r0n5-M$nHfBCylzSM6> zpzg1*d$&=i87X5ZEwAEsC$i@&#b!aAsheYqcq0}e;?Yyo)-jY^{XqE=fgZb$MW}V+ z+3>DRfLGXfhvrVEc`Bis3_aTR6D16%7Z9kA9p)D6z>aCXXjxGPiimvb`hGngoHx>9 z`vMuGjj!t_4$9`kKh{_s;4`_nK@47IfGnkCt=^C=C5|+tYZeuhU)+ov7*r0&5qmjm z&VQHvSJDmW82&^c@#LK`KqVjKFV2{R=apJ3TV zS!HADHR7>hg9CTsoRJ6DT* z&o=@zc<>M8Pc{NuhQRt{Wk(X^6k|+22+Qph25w^0s3-T@qYsqmwSZyxQwat!SivNI z3=NBjsJKV#gL6sN0^pI{DoK=xu|7DF+R10|+^jl5r7H+v4tq5Z(acixnRP!U$Wrw# za>LWPaiYH`$E&lrY3;X=DnU~^P-Q{f(8tjZXqGB0@WfzPlBAWlqkEBErr~SVNNd^o zF4;L>(Mh&PS1pR4`{XM7ZYDHnYfy8?-qFM;-*-*0{?=b(_rCQfYn(*Ac5n>;1pU)U zdk6>Q|C~JfD%w#EK?)^N1wVZ$M#))~wo-2gbAwR}t(#dkkPb*R3z+qz;u~!j^(mxE zyiIkbOE;_h>OB{>@w`@cdl4#Tw-#$6i+!@$GQlAljb^_3$q&sbp&nAY38s$C=+4}T zhYP5;I}_rEM1$=a?+H?!SCURF#g%N}F-pGhd4wF<7iBme-0SL`?>-%$9%MHsD{<$CS^j%tn=Iq8IpBr!}X>ORDuUsJm=E3O2m1``63>5F%t#E6`W|-tFUmHm-N0@fojcGb*@TZWq&) zTk0cNy;5xHuCSoY_%=F~T4o(lCh$7tD`m9v4a;jnw%hbUrmh>N_APImooD2J+hS|X zkxVEQ55&{K2yHF!{N=PAC>no6fi!-UtHn!v#GTI)>d&{uM}1~ zffz5P?g~M}2V)PcL(c;FCcl77wWxCZ{zl%GBdsmd;b4bbH6^!2qa!uZrYE3vJMFXm z-WWy*FKsrh;en}{W~gB(O7YT2P2o788+tUd0Bp)2uTIsZg9Vq`!Oom`-GISBUv)k@ zv5hrSnNf3Lg0scE;P_imv^<)GGTKD#A(7j*MZDH#gPG4w1Qyg#lZ%nYUv}7<+!Zq# zE(Xf(qc7hgO8CWpLtwx++du`B`>QwJCttDVB4!~zs}|nDxm(^iGGrd}@o^7DP1h9% zBCV^cBwgc|g)<*7kYv+CL*(-V4)1u*AY?L2=uEqkYW~__m_cc_!vDGP?;(ZXUQaN^ zsEusz2)q!D#Q%75J>fd$DDdef3;5@!}Hfk`74{5`tfUdKGZ_hSps--tFQkqKUa6U^K6zMA8qiR9OBSH|`L8+hd zHaZR7w^3p!OD--iq!cY)p*I5K+2Cn>8Zga}WwkB5{Ur6JkJ%TBb)Cf~O*hd=923{3 zi$X1jJ3>j4j7i@G@gj`(F;8>FhD*tYxr zolFtV?HR%Pk&H(ygy{f29*`y@fAg>#yS;NTBkq?E9N~+gEo>@;Ln5dsl-ymmPS1`} zsjvEKPZnLJcCE05A@4A9Gz`RCYrQe01usxSg5>K&Tb;9WbjUu@X1kavWy`?AlriZj zlbm6siYQhX<~S&N)!OYm+FRdjBl|=W0v?@ZO>(e(-XUm+ga6ZS1+Gf_2d@Abrm*E2S`MnpnS$ydTFK64Sb(! zKkRKUQ8?vmD7kn{{(>E@@v$h5yuiVp%q+y8AI`KEq4e&|for&swXoR8#G z(4ibL2faT);bEzZeBC#$u}^^$bM;4}p(;WCn8(r6yxSAGbwc$tCh#TP+I(ijOCpWg zdoi0G-)>rSw#>0!!Zm+c`?*-GX>V2gLtejY#OfK;&O6;anP;-6fKch`;-Nn1Xl9>R zE={|MGyup`r38vVU;=AG*khK&S6yV1R9$)&7+>1c6 zvUw#`L)V%2O~E{d@N*mn_BeE5m6WVMON9Ei#98Yp#c}Er&Xe#t|4c(B>g26Ic>pek zkP=t1-}l_)q7qx15h-Tyw@>1@ulU?I%THgcg33hl=E{5|@Z@Y89g;6glgHJ*zX2-a zJOx_**7!@!#J0RHrVf4WitM>~-j3P<#UC5hCnF!?|AsRZL2Li? zFtLU!s^?{ofuFq}$;gEmWEw+R&zpnJ)Sqw_fQf5%XTGH^*nP<}RzQjWzXN zfOOZg2TU}EPDuA@i7OVFmM9{xB>_&Y>I4@+%s%PpKvAO&S%G=_%X4U_UvzA^OagbX zFt8v=eAH7<{6_rdd=B!B?I3_0p+Fc!@NhR7|X zu_Pxl?D&tbP|`6$1)QuDC98iQVY^PU`VO}Li6jNa;-)0N253TJUVd+V)1Vh^qcrKHwV~KERa7U~{)mvm&Ey#6WDV5Ms0d@dI*CAc1Ou`qbAzsoc2*0f6 zL$oxdcfj`P4Ud^JCDNIRnOg|)*7NDxQw*um9&DQ5=lZ^(?oCOPL76xYLI`L6t4rJQM=$R=ewxk@I}2>0xb(W z5`?b0r8^xup<@tgBhavCYeEG0dFrrevZRu+Sy%o1JFp9Ukp9XdO;<+n+zy!uiy?@< zRYU)z*1umhIFKnN-)=)vE7Vd-!2<-q{|tVZgV~L|>Gi1yW-fTpp&w~rYsG%zSX?tv z##*8rX&FmyfWu3-aG{8#b}2nVuwf&ftFfFCMuGTjICnRX5ds0do559HIJ z5I|4*vfbOP{qRfHnfWR}R1aFrbs%5=J$KX51PuX(kmno8{B^Up3;8}v;aQ*+FGK{=ZCna1TcCRX!GGX72tXhij`w8d6) zoELdKI5qNB|BBC`C~lLQj(tDogVlDNquY5Ua~Vc>lQ0Fz4yM-XBg}q(Hhloq?|U4! zjQ~z{2GsrI7Jd=H6mn7X$lKL~x&~qri+>By-kWW-gpJ8VtgnvH*g`>1jIf8NR7vQnhJU{x3IrLKJVm!k# zG0ORIg#=xJ928eK-432is!@34@QG!Qbg2p@?6_nUK<%4M$fjTfClbA?j$Rpk(~k)f z^TXHhTmY6VyhvNVTI5RcBVIU-lxY2TG4t0ew*o$TfwJV;g3wv2boHn_uV`fRBxj{8 z7AUaVj9>o4AMvzm*ren_hZt$pPg=PAWH{jc+?x+Q>GGo0xe~yyetfzPbh$K;{-*si zh|^hFla6MOa`%gLFyY!3Jgoas%cUG?GW3zIUieGTdEqQNy5RE%WfPJed0|`)ak?GX z?$w|Cyl1_6p3?;cYDorr?Mb=;ojcGTuDlVRfV@hzFcQH~Q7A&a_ECfrll&W|Y4yoG zQzw$XcSg8&daWVsC2?B!gj<-uG5GCz3hz(fW{86b#xR=G{BRr13Bdo_?!oIlY6<3M zZPww&wYnhYG&4ltqqJl9OjCbh#E#&T9zN9am~LHl=iySm=^RCF-UFqWd(2U0kX5JJ>2jSnk|5W4$xQlyZKf#cC{OkA&-jUH9H+i1~NQ6k*Pr<_ zLsv_?*v6N-r|@vEUh4)fc*WkIv5s?#s<)d4wL1=>XiCBSOeJR^2rz%d!dHhQR=n!o}bDm@{IJT^jIw!cmh^A1D zte1%(Xp(|`%>_B&Q>m4AG7E?NIQt#1mm=VvwP>F$ebu=g4t(5%5Un`7zGufBK_GPC zM3sTqelRBV3x6^j@H|8amCl!^OM$fq3 zzQMoD@~7572zo)B5>OgBxF+%xApm~0knaMw++jgo(MwrEA%wd9X&`qVBA!XmG^<5s zaFrit39w+A=#I%$m)l5QNDfP_j)|0wD~#y#%k=$97LZ{$PFD4C$3!OT>EyYcGiL+} z2GOK-8CO9;pm(8`i8uyDhL&U|ia=!Mrv)0kY|4F$BsWboTZ6b=xN`U3k6W>#^1O$l%S$fbljOCu6nh*ktI zfM&GXedbFGa{iV8%n=gwBo;9_ZpDp+m?IU*9fX@=oI#4McA zpxHJ26C$K04#iv7_cLWILteIguUBMLk-Odx&AGDU<9`5F`WxIj)|l$9q~1~PTAkUZ zD^CA_KE-}0xFzMIQV)Z;w)7YrlX*X4^}prG;a)kbom_Ul)K!#hxLOAyhM_d@$HyUkjJ$YpNF_oncr z5-VOTm`Q{Z4R6?V{JfcXq3wssHx46Pyl^}=J5dONe^9bcNSJ9}<6jxwNrXXPC`bI( z$Y1_YOOf)?ihx{C?aeVBPgCTcj!vhd6>xtxfMK!Op-&W>(-_YL*n5_hSp?REN^432tpLNo53$A`1n3Se>i5Y~INnG{{fT9v0@MWpI#gFHEYobVZtW+WD`Ri*l4j zsYO9U1o7MQvB~V9f!iVn5$4r)bXl4z7f`qIHgMgE z{#drb0Z-qgIsj^0L4E4aRwU2O$pwnjM%iM4>+G99PsV9WCq!l)wxQU%W1(Sktmw04 zSe;(4Zc+t}s{j6?WT3y#oC6l_ln^5(g`TMQvbuCP0!oLHF&VeE&@`$nnOgzvutm8@ z$;ftqoU)xcwA&qV>>}PZ?Zk*SvrKME;#14=ds_OKSu6H)q1iHqNV0T^aVjV{xgh}{ z5a&h7zGNro3VSRZ?yLi#`@^`>Rf#H6h!cLe*MlYw$joTKPDItKGe=ltcyd8a5lbjr zF-v<{;STNGqYsY-^$>esFfW;5vwYPZ@X}lmd1DUgvEy1_-eTIB`MOBF;j;UdIKPn; zwF8<^a@={7lf>FWW!gxK4W-k@R|L)yGlioYMvxEOuRUC2aJl6jmpXQ!el+JyfB6cl zO=_d|s<;9VDf)?g1p0pkxq~1zk#!SF%Y+4GLfBsBycJTVFMs`ttk^i|6kq3qP^0!Q{K4(zfZ~ zwGvYP?Bw3bFpA>b%@{NA)S@Y*o+amZ*+K}JrlZ2+A4YX6%`KulxLWlilv&a2%Eg?* z8?|MmKy!MR-;-(luKO1yv2?^RTWF{4#1cx?WXLGZ?jj{?$uFg-DeUdTlRf%DcV^ntFGJ58qW0Npj zv`&eYs_%rDrw)c#@Xi`Ym+k>4Qu$?G-D?olhkmMrN2cNk*=)QaVbFr!NZypV#aqr8 zIf%CI-<-Fla@if7+yYX&Q!#1do@NfK2&u-d&VvM%Gn+t|U~yLxQ1H$Ss^YZK))#SWkz*1KmZ;1XJp z#A^)vg&SU2A|mt>mtVqriEv>i7DN$3&qAjs13?ZkQz|SVw!B3U(uxvqY)&-$w(B<2 zGvJwB_t4|hZ2tT4>g z;2aemfqdi0AWT4?qzqOYtQ1CE=}ns4g^{!$RmQT$$(^0S0U-;#FhjKGT1YXW4tqFa znvcWj)h!)PQmbRY@h_H2A$-Pt!tc@w8a4gI>7Av`<5*ynZAJ#H@8+=TG0_zFDa@Mn zdljW9r{t{LW=}zVRjgUUioIPLqWqC;b)Wcv4dRK*{kG9aS>DU)+~@4%&LQ(h)1T$< zGPF&ti%loVZqhyahaD+qh~;UgO-Ltzf8*m#9{Gr%mS$@9DrWpqQ(o^4Q4AN5@YIajxbaQOJTEz5u}yDMWbAeIc;i5{l)JSFmGo@*FAGey z!s8@$G!9=S()iKJtPJ6q=w#^Sv~Kc7lIwP)$8Q;n((Ty>AanGG71XJSNGl-MZXzdzXcc%c_pV*ey8VL{%~@_43MftwEIJ@nxyKRqa)|0z zRAvIbbvIBU#`syXAF}NPIWX$dsw`XHJIe}neT-%yJUYq<`2b}O97^V1(bd7|D&|Ip zNzTSwfdhE#`2l7h(Z>VkUbh?}AeH=FL^o3@+jA>vD!~Ec1!0ThhLz`hu`kDvgPyR~ z5_)n&o%kJAhVZ}-BDBTM0+68=`peg18jA@zE$$p4)@5=JYQ0IdE<{M;4a1K*CczW1 zu)S0q1`c1^n$@ABUPfb5UgZKMq>&?jq0{3n+`806%U`&tZma?hni8}l>MU-I zny5Ym#7Y-(xxjr(PqDazhEp~MnT0t{y#4-FuX?la3UuJTdzjX$!6QH`xrcg64Pm|m zf&3z6ee=2FqhfPU0X%wXxwgHNouWo3J%&H5{aXD z3`3ix$xr!O`oibRK~cw~8yN{Y;-U6PT5D-{a05<27RAW=nqt_@454FChK&{pU9+$~ zAV0t{yDeAokH=^71DhOt1GY#ZC}q@yQpe>Ln60MLU44`XsQaT;TKT^LdiHS3FLqqN zr<9-t2eb7=Yt_S)WZ&mF&u$jS@p-(4Ys4xB0C#nqOYO7QIg#ZIq3r!B#);Bp2}~y2 zIr*&-to-8K7KzuBkaE>Zf0sxFAhY-zdOvTV@??XtgVn(v_bP4eNe7L+O`;2s5l82Q z`k+^nYIPioK}CiAo+AqvzIFZ*AKp*N@Y;&^AIv(dxC302hG_TpN99)x~Oh#u%Yb1{DfdjLsyEqnWOGOfX@w<*dW(uQn=I10Je|K#->lvzl zWMj$Rx)*~5q)as#FW;GQ&Q@N2$&T|^SiKE94$+0EQz(oH<4??Rgw`#ruD=hs9AVIa zE!S=20RxJ>CcF~*(o2{hBqxLmhgwQc9H-BHuDvcgq}E-PDSb#jjQJuSfnOQ=Iq-!N zY+owZASrZU+FAV7!QvqI=+28@ti>QgC91LB;fFQ1+OB1s7B=ClU8W~E4#1V?18>#m z*@23WZ}^E3PZJA0Uy-UqUKGk^3v*^9-8xHMH$y^a53JRLqpY)ED*zA`*$$I~sI&3K zfQz#@*`oj*`I>?k9jx^fW~A`(6ab#v84nb`E+ND z?*}LPVC+JUqnh^J=J>Rkiw(H+Yif{#FDtc1vDAB$+ontb)78F(3XJ9((Za~D_~-aO z^jNj}$AdxN5FF|(j0MrGYOC*I?t<>$Sv|^EL-`(LG^``38}0!eqJ-9~&7s50E95*z z>erH*J2YW5%wDkdZ@;8I_kyL^g%DoGZX|hr_~Uy#o-w5SmoGvX5LqKTGg*pw9aNRP z!fb;foC$hZa}_cwIg)0zpq~$h0_I%m)Z}neoUo=cU*Z+3^P z5^MCD%BNoC(T_RD)k;X&sKbN>S4Xu{3%YupcpIhobe}iCWN_u>B`une~^N*)u#iS+F`B9q4l6IXpP^DKdzq#x+Lee6^$!{&-%5 z+xA5w-&dcZFttI80qV)IBXs_9CzXG_FK(u6deX8kx7fnj|I@n^B-%WDbF4KmS#zSw z>CVyc^F4qWHH|gEU?c1x+Ojz1qU#6yT)+|(AJhdFg(0MlqiL18PCH?t8+oMI(ZpT_ zunTe}(=R2M0fdojC|;?VX(q8d#d@~2=K8uyf^MBreBN9DmYY|(BLbxG!XR(XNB6tF z)9$APG^SK*>O$KaRWob!v&SK=&~fE!J+gk_0|-l zJrKpHqE9$Q0|X#GSO(>~5;)rJPtYw9v*;>|i0GU)*W|I*dc9|8p;?)p2Ek#{a!&ugHn!%^e z5Sd?ZgK?QHRBiz)qlPxM*iajB|7K^-CnD>NbL4ug@0xT*0M3OSWr~VgZB3VP$UGM1 zRhJb$p$SK^-@UQYmcBfS7FWL%NS zWtGCRC$u#x`)_6X?r3G)SkH|1x{KN##*bsD_Netld zx`^B}89=| z?|R#){p_Ar;2lO|XDprtrgxPH1{A$VeP;_Xi`dM8jO=+*iP0blbZRE#nrC4N|VS-Bu%F9qetor0>9ph2o~TbUo7TQ@$G9u;x0c9S%FQR8onf4NQN26IM=hyE*Gvb@i;$k4W}Z}c&hoWJs!L7R;l{hBx4p}{5#)kX8x*oLuX*&{}l0+ zQE>oEmxH@YaECAihr!)taJS$-xF@)V;O;QETd+WIcXxMp0>J{CxBI^Rs!yM~)u+0; z|Moq%Yug@b=U=_DeX1zv11`9)WcyapR%d24qkk3^oObf}N0Tzw(CaPiU(U;t(2NPQ zrKJDpC&{g?JF}Fg@6zuoTW0tmF8(;Re(x~aYPzfMC1n@c`!e#lu*W7-pAW^K*A)=F zV--Zfhs2sYpa&87`&<;>3UY%dK8e+zt6c3eH#^$ARhU*e0aYH!vg?8Q0_|o+$-W>Xvb?W2;3Y%ZY7QfHaSk>)mL-h84UiaG|~7 z^xiNaP7OO?Y4ZD=HrbouIRuMyMgQysNn{p)RGM$PIG%qRZ~IyJy-D-K+FWaZ$cYTs z@vCSGV1Xuh7Dh%LeSkD$)Ie$kgexQdXh>9?687zjnIB_&;4oXAJEdbK+I7IofY^gelLi60xP2k?F5 zO;_F?G09HqJ=7TIhkD?!7e>Wynz8yk+UIGrz;LBvglmR7go?YZ>u!Dvu^4xPc6Pa0 z_|q-DolIY07D0$>Om11}$}Z_-z|pPut$z5)cK=wjbFPwai;NqwikhJn$Eog#!(v6) z#j>@@Iw5(l%)I7GXRM;*%MQMZ>}&6CD!Dv)^iCvj_}#)WEpvU=CvRgA?}STLPU-eW z)-xNjYZHlRyuoX386Ea46FWDxC4RY_U=L07s*2QsUsb4bN7uy0llxEG)f8@#w-J{jFlp6EQ*5K%qbET4Fzl6 zitns=kWf)KJOsZZFgyYdXPdD$q*gr<3U>CzJxi_<(!XfHkPC7J8_vj|2R$03y2kkG z3VSkQJkl(?n~9kuPZ-q}Fz{NHp0TMm4lC3r^PSdUvk|Q~E<9^aGJO01F(^adzp84G zIb_Kb@me@@OQBl)^Sa%=_Bs4s=xAb#olU8KV3e!osX9}ba6g2 zUZjYq8htoy|8OcQ)qposgdwv;T}XTLpR!0L>2L+3l1*-mSh*W>=TI|dCE81V65|1R zg1!pFZ?k45bdp1=R3S|}1&81iq#_Yw*um$IDof?1^Ou4EzyBg)LDwvehv#~$Z;?tk zgX+{9S%mVJAbCjgraT!Yg#c{^dtL@N#wFjSxPAGksRVP;%<6&Fsv+V;dF_ZkWsV)wGd1e&=38#~;|5d)+w^LX-VrXo=bX4kpQB}40;^rxfKYAlG7qZe)$(VjhNhu1r4+@H{I zWcV57F2aC@zU}=Dx!5^YP)c4l9!1l)+YT)=dVV2Psn*?Mb_5_iteMR8=UJC(G`AQX z-us74ltMRi? zQiFh;MOewLmQ?yPQ_o>2Dc?xm*Vg2qGPk-z4{iaJ{L z8ATRXh)BUf&xj(zRG$jqq<+f7;-BSB%V@q~W*90L)Rfnhi98YRCfVw*yK~{picHRaDYRwwJlV=)fyW^{1tP(2sC&lQzvj|3 z?KlD>iyp;?Iy9#4({#O}>{SfV2QDAd=~|Iz8H_P`|FIWdA~)wiX=`4DeiuSZP@7gGsTllC;G z2R(avUupbtxEz^85|fdCTph!lQw-g=4I+^4NEYXABP^xdXpQY|)cVCZ|7makheFc6 zAhjZ69+}d%{6Uz%$^7q%0S!Q=O=7cghKd*&x`YRGp+-(j(iz@-=^mCD`7WI9OtCx} zD#r?xlly|#LX@3OED4oi{3epR%hXQdmA;PP02D6q&;f`o&{=_7qkS|)*s|DhyU346a z+cEXP=xIJ_X_V^T(tB+ZGjk*%!)L_YIbNc9TvVsE;{BMwHVM_ZNBLiaYC3{MS!0mT zAXJtbY|GWQBQForwBgktd``MqaEkvwsHP7*-fQpr>7AgrB{3zb9rBkSfc%5t_t6{W!K&%eG-^vMG@&hxvW6 z$W%6F?;9W7M~Qp(l1Dut4o{6Oo*Gcd!iZDZgK)kb%n_^HvoHRc{sxz;9W2qc^<@{? zZSr)*VSTaLaf6E%%t80Go{~9k*2}Xc%Pdp(wf`L-t1{d6^atRek6ohw*Hz7_&GJ$3eIaL{`9yq7X?;DG z-_@;Q_@OF`L`Dve1uZ{Oxp_zw!%PinPD1e%$I=_OfNDsKiV9A(jjR>ooVEiUK)?1G zG_iO&nY6m?A_#>(ohV9ty2)zsuuf1KRVN8m>B?LA}5o@wRIXEAbFDVk3zUnIG zR~Gk1GJWEkl)|Fhwwkh7Q1E~PRBtYIiUr}ypkW7=Tci~aV5MrL7QeO*+ifV}y*__a z78dgtkhQ&^%Ze+dK-hqY&s<#S;8>M#S6+XHaB4ee5Q@Q0By57HIy4 zXjTeWjwRyPpPIT>(wS!X)`Qzhn2jVRX*h=*w*CtN=7dx~5q%^o44lVxG(R%!ApBf3 zKyv<^B?+nIQ+|w>^Ct>z)eqG16?V*9oq}o8o~^qgr)F)mLz%AnzU^_a!TFPQOof%d z)gq_dxw_UkPWN^5KqumiE!glFIR zed&np6wE{r)g*8-=xvR+6gcwP=t!{tI5yV`B-|v3Q*h@c{&QTly`_H4mTQp-ZV(64v z`>i4am6HR+$a`03Gb?m2jTk4O=;z~CKPwW{8k{pbFQf4*R994PXF%43B(SCQ} zYeD77wHM1qMTo}{;Xl)828{V6OW_v0;M~SpJ-vT z0?#S$9JVf`j(!BHcI{%+=O{ecP>|JyNG$obR^?`fyZOST!r`MH8pWO zGc^Jzypk^I5-addm3_s;IfPub_GU0@YP9`ALBjKkH3$CG0x(0dR|0Cf52gK`HK~O4 z$ZgzR{o^K>j!4k1?vLfsSk&4&XJxdt`Gmwl2+}^w?PS(p7ZUof4I~7Vp>5LkyzkKXjmpOk^0X6J2=-_UelFAkgzK#K3k-HL;qJi%X>T4 zE3M&xfq;G8LaPL&HP&&M1q|ULar}?CXcg3nPde3SRkGA$Bp=_S_Q<&&D_J_lAhGJZ zb%Ft-B`H}_ysyH(yha%EJd}ry??aFCjHLGao4GUMZ5}X?;xvp0@w4lkZ&kmA)|4pZGbb?;n#m(((bhIxuCHTo*?#{&rBMm$(6kldkpmEUqGb6J1a7jrqg)Uc01r+&(zRT}qXDr9j zhqxp113Xv47mW;JWSV}TV%Wm>!8@b+*eQfLNKiH8AxgO;OR-mTz#bA*2iTQ#g#8OY z&yR%~V*lk&W&5apJy(VQ8E-9R`KQxCvzfkHVWL+N9Hka=%2Y=yb;veNZ)>=`CyC_&TLY(}o; z&tdS5ui4TIe?1ZD_$s06mRV?h8y}5~gmSI9{h~cM)t~DXvOsprpBjs&Du65r)}6-J zPWG5my=`olnh0?o*1LhoOj{>8UYw6hpReQ0Wg>*Bb#^ESo=x_`$w_6QTkRy_l7Zsb)eTkgJrP)v^%E2V`%nMy3QK;0%pr4lx%NC7s9 zV>SSJv`zq1-HlEa>Ufvluh!OUo?|!AfM@szgT*(|gxqw1idLt%e_=vf1o3<-G{N?? zndQj~Gr)C}vo|i-2%o3sVjwA?JB%9p&R1PAic?CH?i}E&7q+^|JJ_I&#%l96W*R!Y z8!N2Nko(MkKQ{zVb6gWmXe)1+{kxL2Zux^cKq$YB(%o|lxofle>%6gxU4!>&)Py%r z^!_(t4CU_)2}xvQ`ZifmfH?gUJS#Y3jX;9+nlH79135EL5(Mh-lii z>(BrKfC7+tw8 zjrl`74ui+vDO0fcc-akBWa8-Ww=^Zqj0eUJ7pTj&A7436Cj3lNd*OwtR$Tx@x>+~E zC2ku^isW5pu3!b(k4i`GHa{m=6)t*zwRxgsp_45fmDmE;u_}@V{4PiJsl8YRTH|Sf zd@=lL$dseF_b_uOEMI3)e+FB}NT1Nu3F%skU%vdNq%=B*3>Sh+mQHPvl<7nUSu=hE zVddhr9pXu1#ANia#R z&mVxYLb+6I20r_>Ow+k`cO2ut=VLp6|CK>;^%gL5zGw?VAQj+jdWU!h* zpgSCs&jK1Bj3W&`zAC&tD(r*@ADk36lRFn{k@1jLnk+63al&Z2w*lK#TbM-3ug%De z&r^Zy-_*V1H22WaAsDi7I8U^h@28_)Cwe;)$+LB0uDo`@$7#xGc!N_ixdLVzT(s6Y< zyeYi+F)$8b1J4}MH2q`|_#-)eTOpPQp^{ho$h*MlGipSe=z0jRVUq1!1X1sux`EMB z^-Ttj>H4xrSc@n36r5Vo!tc@dE1tT!yvHFOE9{S@1zC_OTsPqww$2^J%5UE{g2%gSZ>6VR`0?*PMaS z%kCqx(&}W=??em^bQC9&1Qx*Zf+_nS(@%;sTg5N$?bKkU+@p~!7s?&nhRehFj32Y4 zgcQ1;(;o#%V&Z=(F4C^>+0=$w1RiYUo6O|`t5UBhQ)gIVbUlgPmDp!C%!Lw^mRDbA z(Xv^S1nf1_T~;rtg%?i(RDRWTJy}fp@6pVgjH2E7Ml`hC_C8hg9z1Sy;yy#4M-Z)s z0G6*_J)A8VH=*0qcMV^QR#D8VttjI1K6iAxE91Au+LI)F{PbYslND*rPD^L#Sxy1F zyjTbnU)*ZbGEG=;EYGllUS>};AeZ~}At<@=h(}VpJU74gtZ5PPb##%~phNHBM2)ki z6l@0uE@!FE`A=N4_?q7lr-S%`*=QZG@i@ubSzxzfE4{r&te~`&8LL?b>e}Wsy@&z- zACsOFok=g^C#+Gw=;$EIq*F6uYtnnFZZw2mIaO7pI=tDm@|SY2j^RTwNNFxqY)|WW zBMw@9oQOS5c|%R|LMiEKK40^lS{Lcl0;p>X>1F5Ui2psYo(Ac2r1Wm*n4{@ch9An- zncL+NMQU&z9C{5LQKN{qe<>Cgo^!icy+?VW=V*P>&CT@7&@sQ=(0e=QO+nx;GEDY{ zQh;06@`gxh1|8BsnJ5h?X1Uk*lDq{)42^Q^(q>bJR!PaLJ+UPx!^^P34D%E>^a3$8 z2}@rGejhZK8Q;t|W0zGtb;C(?@rfAmqJTY>Jg@!g+6~3V@FP7nfH2eBQbcVT8wes8 zghG=D5S0CIiO{PqgzqfD>NZ0dC3p6x-Z$|zYr&;cT#Ln|au?+kmiV|^flje8gUsNY z4zGa#3dEzN3oER*sVcgjvU$Pv5%*vFft3H$#aUA8?_O2Pl!=`8YCE~^)8|H7S74@6 z|Ai27>kL>lO`i`f%y3=#T{>RZ<5~{>on$6%dMwph1iaG6dAVBZo$Yt$b`+TdJGiQP z@^M+9e@&4-j|Xcq+1k-rp4`ZO7gf5DO$dr{eVA|{Z>#LKHmfFwLt8tyu6IftWG%#5 z4>WKwcM9@BM>7ts6Hc3rT#VNWi%#Mv={bvfIvn>JAF zpV$>Cx06-&Gg17!dz@~V5YAe*{_%H3Ngs;Q8yT<8<7ri7uX@u;K)5v1_Q0R~fini!HQXSo$@h5e#b~@L4aV zn1Tale@KMP>MDSBZpeIY#)jbc^2?3B>m^~{*WxVN$S*&(nONhOY3$~S0-`wSr|UE% zyV~f|j@A2q9vxgwy1(&CCJy9CTK8sT31Z(`j_ltp&B@y%f5;wx*>#DVW~fEE`DEPa zv^F*Y3Y=J@HAvO2&8y2}nk64CYNx_3ArF#cXb_1Lh`Rv6F*udvH_Kz_EKaDW=SB1Q9#zKxtYrr5@S!$Z*8X10O-+K60ii=VqOs+w?S zB+74w$v={cT%#KUsAv}Kl-L@n2umD22T?h#&OTA6jcV0sZ7~x2tM6={4A*of6T=jK zgFFRS7oOD+8atU?HqM1hi#(<3QaOj|7!8<5X`vN^c6^kaI3xUi^&8F?Dn z{#5WY$3R7sqX$QsQMsvMDoGPMky=%qbcdn71AG4O_!q3ermMg}9HJch=cpw?Xc+PF*fC-P8vLAv5+KqQ#8aDRN2hZd9=Kj#)XJ zalbO52h9PRi!sxNGEZV{qdULsrsSH{Ycn>L%Ff+txku@P4kY}``s=q^7eQq^w$(^u zOd{_iB`x$CB4=NmKuZSQ3g5t_?dYB9_C?chN3QVS=k`HB42o2$p|cgz)4)yxL)cug4pV37t z$FKQt0vFmWzx*2H%}v~?!ESjbC=D-!dJTb@jNVKM3RDO(#T|lSlhyF9?9;&-{>QL}y^FdlRslYabhCMO~5OSFWT<3DZ9-_tN#Ej{q16Y#WKb3+8!qwu>mW zAZbKvQWM6ox&l}q$-HRd#lL*;V#IR@GNg5Z0D!cvyiD7_Hi**RwpIXuk`jRFUw{li zfWZa8{X;PS1^|Wxfbf4Y0H6Rv`v0*y4C8-uU;%&#TLAojbM*iD|Crpr`EU1sJyJXX zfb=hdXXoVPNPzi&7zP#&0R5kJ4t91Z%>SMWh5et3ur5%z|HVa!|IPVV>_7H@01=09 A6#xJL literal 0 HcmV?d00001 From 74223b3467fe71acd3f64fc5f175cc65ddbbc24b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 11 Nov 2022 15:22:13 +0800 Subject: [PATCH 27/46] more optimize --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 205 ++++++++++++++++++++++--- 1 file changed, 187 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index a40b1667c5..bfdc8ad4eb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1015,6 +1015,182 @@ _err: return code; } +static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) { + int32_t code = 0; + + SColVal cv = {0}; + int32_t iColDataFrom = 0; + SColData *pColDataFrom = + (iColDataFrom < pBlockDataFrom->nColData) ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] : NULL; + + for (int32_t iColDataTo = 0; iColDataTo < pBlockData->nColData; iColDataTo++) { + SColData *pColDataTo = &((SColData *)pBlockData->aColData->pData)[iColDataTo]; + + while (pColDataFrom && pColDataFrom->cid < pColDataTo->cid) { + iColDataFrom++; + pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) + ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] + : NULL; + } + + if (pColDataFrom == NULL || pColDataFrom->cid > pColDataTo->cid) { + code = tColDataAppendValue(pColDataTo, &COL_VAL_NONE(pColDataTo->cid, pColDataTo->type)); + if (code) goto _exit; + } else { + tColDataGetValue(pColDataFrom, iRow, &cv); + + code = tColDataAppendValue(pColDataTo, &cv); + if (code) goto _exit; + + iColDataFrom++; + pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) + ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] + : NULL; + } + } + +_exit: + return code; +} + +static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { + int32_t code = 0; + + int32_t iTColumn = 1; + STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; + + while (pTColumn && pTColumn->colId < pColData->cid) { + iTColumn++; + pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + + if (pTColumn == NULL || pTColumn->colId > pColData->cid) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + if (code) goto _exit; + } else { + ASSERT(pTColumn->type == pColData->type); + + SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; + + if (pRow->statis) { + TDRowValT vt; + tdGetBitmapValTypeII(tdGetBitmapAddrTp(pRow, pTSchema->flen), iTColumn - 1, &vt); + + if (vt == TD_VTYPE_NORM) { + cv.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + cv.value.nData = varDataLen(pData); + cv.value.pData = varDataVal(pData); + } else { + memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + } + + code = tColDataAppendValue(pColData, &cv); + if (code) goto _exit; + } else if (vt == TD_VTYPE_NONE) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + if (code) goto _exit; + } else if (vt == TD_VTYPE_NULL) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); + if (code) goto _exit; + } else { + ASSERT(0); + } + } else { + cv.flag = CV_FLAG_VALUE; + + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + cv.value.nData = varDataLen(pData); + cv.value.pData = varDataVal(pData); + } else { + memcpy(&cv.value.val, pRow->data + pTColumn->offset - sizeof(TSKEY), pTColumn->bytes); + } + + code = tColDataAppendValue(pColData, &cv); + if (code) goto _exit; + } + + iTColumn++; + pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } + +_exit: + return code; +} + +static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { + int32_t code = 0; + + int32_t iTColumn = 1; + STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; + + while (pTColumn && pTColumn->colId < pColData->cid) { + iTColumn++; + pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + + if (pTColumn == NULL || pTColumn->colId > pColData->cid) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + if (code) goto _exit; + } else { + ASSERT(pTColumn->type == pColData->type); + + // SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; + + // if (pRow->statis) { + // TDRowValT vt; + // tdGetBitmapValTypeII(tdGetBitmapAddrTp(pRow, pTSchema->flen), iTColumn - 1, &vt); + + // if (vt == TD_VTYPE_NORM) { + // cv.flag = CV_FLAG_VALUE; + + // if (IS_VAR_DATA_TYPE(pTColumn->type)) { + // void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset); + // cv.value.nData = varDataLen(pData); + // cv.value.pData = varDataVal(pData); + // } else { + // memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); + // } + + // code = tColDataAppendValue(pColData, &cv); + // if (code) goto _exit; + // } else if (vt == TD_VTYPE_NONE) { + // code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + // if (code) goto _exit; + // } else if (vt == TD_VTYPE_NULL) { + // code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); + // if (code) goto _exit; + // } else { + // ASSERT(0); + // } + // } else { + // cv.flag = CV_FLAG_VALUE; + + // if (IS_VAR_DATA_TYPE(pTColumn->type)) { + // void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset); + // cv.value.nData = varDataLen(pData); + // cv.value.pData = varDataVal(pData); + // } else { + // memcpy(&cv.value.val, pRow->data + pTColumn->offset, pT + // } + // } + } + } + +_exit: + return code; +} + int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { int32_t code = 0; @@ -1036,27 +1212,20 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS if (code) goto _err; pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); - // OTHER - SRowIter rIter = {0}; - SColVal *pColVal; - - tRowIterInit(&rIter, pRow, pTSchema); - pColVal = tRowIterNext(&rIter); - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pColVal && pColVal->cid < pColData->cid) { - pColVal = tRowIterNext(&rIter); - } - - if (pColVal == NULL || pColVal->cid > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + SColVal cv = {0}; + if (pRow->type == 0) { + if (TD_IS_TP_ROW(pRow->pTSRow)) { + code = tBlockDataAppendTPRow(pBlockData, pRow->pTSRow, pTSchema); + if (code) goto _err; + } else if (TD_IS_KV_ROW(pRow->pTSRow)) { + code = tBlockDataAppendKVRow(pBlockData, pRow->pTSRow, pTSchema); if (code) goto _err; } else { - code = tColDataAppendValue(pColData, pColVal); - if (code) goto _err; - pColVal = tRowIterNext(&rIter); + ASSERT(0); } + } else { + code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow); + if (code) goto _err; } pBlockData->nRow++; From 265bb81b49b7a5013faad078d5c24fb0fda3cdff Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 11 Nov 2022 15:35:17 +0800 Subject: [PATCH 28/46] fix(query): set offset for empty record --- source/libs/executor/src/scanoperator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 945753cc53..a3dc128ced 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1773,6 +1773,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { + tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer); return NULL; } ASSERT(pInfo->tqReader->pWalReader->curVersion == pTaskInfo->streamInfo.snapshotVer + 1); From 3b68771a42635719bf42243508eb87bcc1e929b0 Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Fri, 11 Nov 2022 15:44:55 +0800 Subject: [PATCH 29/46] fix: add html close tag for compile --- docs/zh/05-get-started/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index cbdc332477..33661580ee 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -18,4 +18,4 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; 微信扫描下面二维码,加“小 T”为好友,即可加入“物联网大数据技术前沿群”,与大家共同交流物联网大数据技术应用、TDengine 使用问题和技巧等话题。 - + From 41a2b0e8eaa1439bb0dcf5b59757d2ec524652ad Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 16:01:41 +0800 Subject: [PATCH 30/46] chore: append kv row to block data --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 81 ++++++++++++++------------ 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index bfdc8ad4eb..d58c9dab84 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1058,6 +1058,7 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch int32_t iTColumn = 1; STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; @@ -1076,8 +1077,8 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; if (pRow->statis) { - TDRowValT vt; - tdGetBitmapValTypeII(tdGetBitmapAddrTp(pRow, pTSchema->flen), iTColumn - 1, &vt); + TDRowValT vt = TD_VTYPE_MAX; + tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); if (vt == TD_VTYPE_NORM) { cv.flag = CV_FLAG_VALUE; @@ -1128,6 +1129,10 @@ _exit: static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { int32_t code = 0; + col_id_t kvIter = 0; + col_id_t nKvCols = tdRowGetNCols(pRow) - 1; + void *pColIdx = TD_ROW_COL_IDX(pRow); + void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); int32_t iTColumn = 1; STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; @@ -1144,46 +1149,48 @@ static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSch if (code) goto _exit; } else { ASSERT(pTColumn->type == pColData->type); + ASSERT(pTColumn->colId == pColData->cid); - // SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; + SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; + TDRowValT vt = TD_VTYPE_NONE; // default is NONE + SKvRowIdx *pKvIdx = NULL; - // if (pRow->statis) { - // TDRowValT vt; - // tdGetBitmapValTypeII(tdGetBitmapAddrTp(pRow, pTSchema->flen), iTColumn - 1, &vt); + while (kvIter < nKvCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); + if (pKvIdx->colId == pTColumn->colId) { + tdGetBitmapValTypeII(pBitmap, kvIter, &vt); + ++kvIter; + break; + } else if (pKvIdx->colId > pTColumn->colId) { + vt = TD_VTYPE_NONE; + break; + } else { + ++kvIter; + } + } - // if (vt == TD_VTYPE_NORM) { - // cv.flag = CV_FLAG_VALUE; + if (vt == TD_VTYPE_NORM) { + cv.flag = CV_FLAG_VALUE; - // if (IS_VAR_DATA_TYPE(pTColumn->type)) { - // void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset); - // cv.value.nData = varDataLen(pData); - // cv.value.pData = varDataVal(pData); - // } else { - // memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - // } + void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + cv.value.nData = varDataLen(pData); + cv.value.pData = varDataVal(pData); + } else { + memcpy(&cv.value.val, pData, pTColumn->bytes); + } - // code = tColDataAppendValue(pColData, &cv); - // if (code) goto _exit; - // } else if (vt == TD_VTYPE_NONE) { - // code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - // if (code) goto _exit; - // } else if (vt == TD_VTYPE_NULL) { - // code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - // if (code) goto _exit; - // } else { - // ASSERT(0); - // } - // } else { - // cv.flag = CV_FLAG_VALUE; - - // if (IS_VAR_DATA_TYPE(pTColumn->type)) { - // void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset); - // cv.value.nData = varDataLen(pData); - // cv.value.pData = varDataVal(pData); - // } else { - // memcpy(&cv.value.val, pRow->data + pTColumn->offset, pT - // } - // } + code = tColDataAppendValue(pColData, &cv); + if (code) goto _exit; + } else if (vt == TD_VTYPE_NONE) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + if (code) goto _exit; + } else if (vt == TD_VTYPE_NULL) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); + if (code) goto _exit; + } else { + ASSERT(0); + } } } From 50f148abac116102f6cd517e8cd337097fa40ba9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Nov 2022 16:04:20 +0800 Subject: [PATCH 31/46] conn timeout refactor --- source/libs/transport/src/transCli.c | 52 +++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3af0747bcf..a5955beaca 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -27,7 +27,6 @@ typedef struct SCliConn { queue wreqQueue; uv_timer_t* timer; // read timer, forbidden - uv_timer_t connTimer; void* hostThrd; @@ -468,8 +467,15 @@ void cliHandleExcept(SCliConn* conn) { void cliConnTimeout(uv_timer_t* handle) { SCliConn* conn = handle->data; + SCliThrd* pThrd = conn->hostThrd; + tTrace("%s conn %p conn timeout, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + uv_timer_stop(handle); + handle->data = NULL; + taosArrayPush(pThrd->timerList, &conn->timer); + + conn->timer = NULL; cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); } void cliReadTimeoutCb(uv_timer_t* handle) { @@ -648,8 +654,14 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); conn->stream->data = conn; - uv_timer_init(pThrd->loop, &conn->connTimer); - conn->connTimer.data = conn; + uv_timer_t* timer = taosArrayGetSize(pThrd->timerList) > 0 ? *(uv_timer_t**)taosArrayPop(pThrd->timerList) : NULL; + if (timer == NULL) { + timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + tDebug("no available timer, create a timer %p", timer); + uv_timer_init(pThrd->loop, timer); + } + timer->data = conn; + conn->timer = timer; conn->connReq.data = conn; transReqQueueInit(&conn->wreqQueue); @@ -831,7 +843,12 @@ _RETURN: void cliConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; - uv_timer_stop(&pConn->connTimer); + SCliThrd* pThrd = pConn->hostThrd; + + uv_timer_stop(pConn->timer); + pConn->timer->data = NULL; + taosArrayPush(pThrd->timerList, &pConn->timer); + pConn->timer = NULL; if (status != 0) { tError("%s conn %p failed to connect server:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_strerror(status)); @@ -1021,11 +1038,16 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { if (ret != 0) { tTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, uv_err_name(ret)); - uv_timer_stop(&conn->connTimer); + + uv_timer_stop(conn->timer); + conn->timer->data = NULL; + taosArrayPush(pThrd->timerList, &conn->timer); + conn->timer = NULL; + cliHandleExcept(conn); return; } - uv_timer_start(&conn->connTimer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0); + uv_timer_start(conn->timer, cliConnTimeout, TRANS_CONN_TIMEOUT, 0); } STraceId* trace = &pMsg->msg.info.traceId; tGTrace("%s conn %p ready", pTransInst->label, conn); @@ -1148,6 +1170,8 @@ static void* cliWorkThread(void* arg) { pThrd->pid = taosGetSelfPthreadId(); setThreadName("trans-cli-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); + + tDebug("thread quit-thread:%08" PRId64, pThrd->pid); return NULL; } @@ -1207,7 +1231,7 @@ static SCliThrd* createThrdObj(void* trans) { pThrd->prepare->data = pThrd; // uv_prepare_start(pThrd->prepare, cliPrepareCb); - int32_t timerSize = 512; + int32_t timerSize = 64; pThrd->timerList = taosArrayInit(timerSize, sizeof(void*)); for (int i = 0; i < timerSize; i++) { uv_timer_t* timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); @@ -1241,6 +1265,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { transDQDestroy(pThrd->delayQueue, destroyCmsg); transDQDestroy(pThrd->timeoutQueue, NULL); + tDebug("thread destroy %" PRId64, pThrd->pid); for (int i = 0; i < taosArrayGetSize(pThrd->timerList); i++) { uv_timer_t* timer = taosArrayGetP(pThrd->timerList, i); taosMemoryFree(timer); @@ -1266,7 +1291,18 @@ void cliSendQuit(SCliThrd* thrd) { } void cliWalkCb(uv_handle_t* handle, void* arg) { if (!uv_is_closing(handle)) { - uv_read_stop((uv_stream_t*)handle); + if (uv_handle_get_type(handle) == UV_TIMER) { + // SCliConn* pConn = handle->data; + // if (pConn != NULL && pConn->timer != NULL) { + // SCliThrd* pThrd = pConn->hostThrd; + // uv_timer_stop((uv_timer_t*)handle); + // handle->data = NULL; + // taosArrayPush(pThrd->timerList, &pConn->timer); + // pConn->timer = NULL; + // } + } else { + uv_read_stop((uv_stream_t*)handle); + } uv_close(handle, cliDestroy); } } From 7ea790021fb021bf46ed35491a32ba094ec3073b Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 11 Nov 2022 16:15:16 +0800 Subject: [PATCH 32/46] fix:avoid duplicate results --- source/libs/executor/src/executorimpl.c | 1 + source/libs/executor/src/timewindowoperator.c | 35 +++- source/libs/stream/src/streamState.c | 10 +- tests/script/tsim/stream/state0.sim | 188 ++++++++++++++++++ 4 files changed, 220 insertions(+), 14 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 34f462cb3d..7fd288cd57 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3423,6 +3423,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta ASSERT(code == 0); if (code == -1) { // coverity scan + pGroupResInfo->index += 1; continue; } SResultRow* pRow = (SResultRow*)pVal; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4c369e8802..40742087ea 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3580,6 +3580,11 @@ static void removeSessionResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessio tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); } +static void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) { + *pHashKey = *pKey; + pHashKey->win.ekey = pKey->win.skey; +} + static void removeSessionResults(SSHashObj* pHashMap, SArray* pWins) { if (tSimpleHashGetSize(pHashMap) == 0) { return; @@ -3588,8 +3593,8 @@ static void removeSessionResults(SSHashObj* pHashMap, SArray* pWins) { for (int32_t i = 0; i < size; i++) { SSessionKey* pWin = taosArrayGet(pWins, i); if (!pWin) continue; - SSessionKey key = *pWin; - key.win.ekey = key.win.skey; + SSessionKey key = {0}; + getSessionHashKey(pWin, &key); tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); } } @@ -3642,7 +3647,9 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { streamStateSessionDel(pAggSup->pState, pKey); - tSimpleHashRemove(pAggSup->pResultRows, pKey, sizeof(SSessionKey)); + SSessionKey hashKey = {0}; + getSessionHashKey(pKey, &hashKey); + tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey)); return true; } @@ -3753,8 +3760,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - SSessionKey key = winInfo.sessionWin; - key.win.ekey = key.win.skey; + SSessionKey key = {0}; + getSessionHashKey(&winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); } @@ -3896,8 +3903,8 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; - SSessionKey chWinKey = *pWinKey; - chWinKey.win.ekey = chWinKey.win.skey; + SSessionKey chWinKey = {0}; + getSessionHashKey(pWinKey, &chWinKey); SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey); SResultRow* pResult = NULL; SResultRow* pChResult = NULL; @@ -3978,8 +3985,8 @@ static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { for (int32_t i = 0; i < size; i++) { SSessionKey* pWinKey = taosArrayGet(pResWins, i); if (!pWinKey) continue; - SSessionKey winInfo = *pWinKey; - winInfo.win.ekey = winInfo.win.skey; + SSessionKey winInfo = {0}; + getSessionHashKey(pWinKey, &winInfo); tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); } } @@ -4561,8 +4568,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - SSessionKey key = curWin.winInfo.sessionWin; - key.win.ekey = key.win.skey; + SSessionKey key = {0}; + getSessionHashKey(&curWin.winInfo.sessionWin, &key); tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); } } @@ -4645,6 +4652,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); +#if 0 + char* pBuf = streamStateSessionDump(pInfo->streamAggSup.pState); + qDebug("===stream===final session%s", pBuf); + taosMemoryFree(pBuf); +#endif + doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "single state delete"); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ccb0dd4a92..88c39c1157 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -521,9 +521,13 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa void* tmp = NULL; int32_t code = streamStateSessionGetKVByCur(pCur, &resKey, &tmp, pVLen); if (code == 0) { - *key = resKey; - *pVal = tdbRealloc(NULL, *pVLen); - memcpy(*pVal, tmp, *pVLen); + if (key->win.skey != resKey.win.skey) { + code = -1; + } else { + *key = resKey; + *pVal = tdbRealloc(NULL, *pVLen); + memcpy(*pVal, tmp, *pVLen); + } } streamStateFreeCur(pCur); return code; diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index dc7d9bc407..87d7d4b7fc 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -544,4 +544,192 @@ if $rows != 10 then endi +sql drop stream if exists streams4; +sql drop database if exists test4; +sql drop stable if exists streamt4; +sql create database if not exists test4 vgroups 10 precision "ms" ; +sql use test4; +sql create table st (ts timestamp, c1 tinyint, c2 smallint) tags (t1 tinyint) ; +sql create table t1 using st tags (-81) ; +sql create table t2 using st tags (-81) ; +sql create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS start, min(c1),count(c1) from t1 state_window(c1); + +sql insert into t1 (ts, c1) values (1668073288209, 11); +sql insert into t1 (ts, c1) values (1668073288210, 11); +sql insert into t1 (ts, c1) values (1668073288211, 11); +sql insert into t1 (ts, c1) values (1668073288212, 11); +sql insert into t1 (ts, c1) values (1668073288213, 11); +sql insert into t1 (ts, c1) values (1668073288214, 11); +sql insert into t1 (ts, c1) values (1668073288215, 29); + +$loop_count = 0 +loop7: + +sleep 200 + +sql select * from streamt4 order by start; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +if $data01 != 11 then + print =====data01=$data01 + goto loop7 +endi + +if $data02 != 6 then + print =====data02=$data02 + goto loop7 +endi + +sql delete from t1 where ts = cast(1668073288214 as timestamp); +sql insert into t1 (ts, c1) values (1668073288216, 29); +sql delete from t1 where ts = cast(1668073288215 as timestamp); +sql insert into t1 (ts, c1) values (1668073288217, 29); +sql delete from t1 where ts = cast(1668073288216 as timestamp); +sql insert into t1 (ts, c1) values (1668073288218, 29); +sql delete from t1 where ts = cast(1668073288217 as timestamp); +sql insert into t1 (ts, c1) values (1668073288219, 29); +sql delete from t1 where ts = cast(1668073288218 as timestamp); +sql insert into t1 (ts, c1) values (1668073288220, 29); +sql delete from t1 where ts = cast(1668073288219 as timestamp); + +$loop_count = 0 +loop8: + +sleep 200 + +sql select * from streamt4 order by start; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop8 +endi + +if $data01 != 11 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != 5 then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 (ts, c1) values (1668073288221, 65); +sql insert into t1 (ts, c1) values (1668073288222, 65); +sql insert into t1 (ts, c1) values (1668073288223, 65); +sql insert into t1 (ts, c1) values (1668073288224, 65); +sql insert into t1 (ts, c1) values (1668073288225, 65); +sql insert into t1 (ts, c1) values (1668073288226, 65); + +$loop_count = 0 +loop8: + +sleep 200 + +sql select * from streamt4 order by start; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop8 +endi + +if $data01 != 11 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != 5 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 29 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop8 +endi + +sql insert into t1 (ts, c1) values (1668073288224, 64); + +$loop_count = 0 +loop9: + +sleep 200 + +sql select * from streamt4 order by start; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 4 then + print =====rows=$rows + goto loop9 +endi + +if $data01 != 11 then + print =====data01=$data01 + goto loop9 +endi + +if $data02 != 5 then + print =====data02=$data02 + goto loop9 +endi + +if $data11 != 29 then + print =====data11=$data11 + goto loop9 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop9 +endi + +if $data21 != 65 then + print =====data21=$data21 + goto loop9 +endi + +if $data22 != 3 then + print =====data22=$data22 + goto loop9 +endi + +if $data31 != 64 then + print =====data31=$data31 + goto loop9 +endi + +if $data32 != 1 then + print =====data32=$data32 + goto loop9 +endi + + system sh/exec.sh -n dnode1 -s stop -x SIGINT From e8e1d64a9d16c68610c2f8b63174909aacfd9105 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 11 Nov 2022 16:28:02 +0800 Subject: [PATCH 33/46] Update 10-cpp.mdx --- docs/zh/08-connector/10-cpp.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/08-connector/10-cpp.mdx b/docs/zh/08-connector/10-cpp.mdx index cc7991da74..8a4f4946a7 100644 --- a/docs/zh/08-connector/10-cpp.mdx +++ b/docs/zh/08-connector/10-cpp.mdx @@ -115,6 +115,7 @@ TDengine 客户端驱动的安装请参考 [安装指南](../#安装步骤) 订阅和消费 ```c + {{#include examples/c/tmq.c}} ``` From 0ee7ccaf71581ffe615696c6918eca2d926cf58b Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Fri, 11 Nov 2022 16:30:39 +0800 Subject: [PATCH 34/46] fix: import image before using in md/mdx file --- docs/zh/05-get-started/index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 33661580ee..dec4d800bc 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -3,6 +3,8 @@ title: 立即开始 description: '快速设置 TDengine 环境并体验其高效写入和查询' --- +import xiaot from './tdengine.webp' + TDengine 完整的软件包包括服务端(taosd)、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动(taosc)、命令行程序 (CLI,taos) 和一些工具软件。TDengine 除了提供多种语言的连接器之外,还通过 [taosAdapter](../reference/taosadapter) 提供 [RESTful 接口](../connector/rest-api)。 本章主要介绍如何利用 Docker 或者安装包快速设置 TDengine 环境并体验其高效写入和查询。 @@ -18,4 +20,4 @@ import {useCurrentSidebarCategory} from '@docusaurus/theme-common'; 微信扫描下面二维码,加“小 T”为好友,即可加入“物联网大数据技术前沿群”,与大家共同交流物联网大数据技术应用、TDengine 使用问题和技巧等话题。 - +小 T 的二维码 From 97e83a905b04425ab8f9682900eecc5a885f507d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 17:07:16 +0800 Subject: [PATCH 35/46] fix: iter next column for kv row --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d58c9dab84..3387d72a5d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1191,6 +1191,9 @@ static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSch } else { ASSERT(0); } + + iTColumn++; + pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; } } From 9aeac303598e96451aad6bc49b840680165d66fa Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Fri, 11 Nov 2022 17:25:57 +0800 Subject: [PATCH 36/46] docs: optimize search result of download --- docs/zh/28-releases/01-tdengine.md | 2 ++ docs/zh/28-releases/02-tools.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 31093ce557..83ca096d32 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -4,6 +4,8 @@ title: TDengine 发布历史 description: TDengine 发布历史、Release Notes 及下载链接 --- +各版本 TDengine 安装包下载链接如下: + import Release from "/components/ReleaseV3"; ## 3.0.1.6 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 2623391fb9..2152dede93 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -4,6 +4,8 @@ title: taosTools 发布历史 description: taosTools 的发布历史、Release Notes 和下载链接 --- +各版本 taosTools 安装包下载链接如下: + import Release from "/components/ReleaseV3"; ## 2.2.7 From f56148a450691cef2cbb0d5db609f67c6c27affd Mon Sep 17 00:00:00 2001 From: Pan YANG Date: Fri, 11 Nov 2022 17:35:08 +0800 Subject: [PATCH 37/46] docs: add download keyword to title --- docs/zh/28-releases/01-tdengine.md | 3 +-- docs/zh/28-releases/02-tools.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 83ca096d32..f72735d903 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -1,6 +1,6 @@ --- sidebar_label: TDengine 发布历史 -title: TDengine 发布历史 +title: TDengine 发布历史及下载链接 description: TDengine 发布历史、Release Notes 及下载链接 --- @@ -35,4 +35,3 @@ import Release from "/components/ReleaseV3"; ## 3.0.1.0 - diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 2152dede93..ac4a884f8b 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -1,6 +1,6 @@ --- sidebar_label: taosTools 发布历史 -title: taosTools 发布历史 +title: taosTools 发布历史及下载链接 description: taosTools 的发布历史、Release Notes 和下载链接 --- From 9dae937017284900fea6c95e8d87d7064f2293b9 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Fri, 11 Nov 2022 17:41:47 +0800 Subject: [PATCH 38/46] test: add log output --- tests/system-test/7-tmq/tmqShow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index 406fd9f7f3..5e79f17459 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -137,7 +137,7 @@ class TDTestCase: tdLog.info("check show subscriptions") tdSql.query("show subscriptions") - # tdLog.debug(tdSql.queryResult) + tdLog.debug(tdSql.queryResult) rows = tdSql.getRows() expectSubscriptions = paraDict['vgroups'] * len(topicNameList) tdLog.info("show subscriptions rows: %d, expect Subscriptions: %d"%(rows,expectSubscriptions)) From 0a298fb23dc3e3d3176613948338ebac7ce83b4c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 11 Nov 2022 18:06:41 +0800 Subject: [PATCH 39/46] fix(query): set tbname for stream session delete --- source/libs/executor/src/timewindowoperator.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4c369e8802..dab670016f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3853,23 +3853,28 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo colDataAppendNULL(pCalEdCol, pBlock->info.rows); SHashObj* pGroupIdTbNameMap = NULL; - if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || + if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOp->info; pGroupIdTbNameMap = pInfo->pGroupIdTbNameMap; } else if (pOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOp->info; pGroupIdTbNameMap = pInfo->pGroupIdTbNameMap; + } else { + ASSERT(0); } char* tbname = taosHashGet(pGroupIdTbNameMap, &res->groupId, sizeof(int64_t)); SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX); if (tbname == NULL) { + /*printf("\n\n no tbname for group id %" PRId64 "%p %p\n\n", res->groupId, pOp->info, pGroupIdTbNameMap);*/ colDataAppendNULL(pTableCol, pBlock->info.rows); } else { char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN]; STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName)); colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false); + /*printf("\n\n get tbname %s group id %" PRId64 "\n\n", tbname, res->groupId);*/ } pBlock->info.rows += 1; } @@ -4046,7 +4051,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pBlock->info.parTbName[0]) { taosHashPut(pInfo->pGroupIdTbNameMap, &pBlock->info.groupId, sizeof(int64_t), &pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); - /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ + /*printf("\n\n put tbname %s group id %" PRId64 "\n\n into %p %p", pBlock->info.parTbName, pBlock->info.groupId,*/ + /*pInfo, pInfo->pGroupIdTbNameMap);*/ } if (pBlock->info.parTbName[0]) { From 0dcbe92006c80ec0bec19974c3b574cdd6d582e6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 19:04:54 +0800 Subject: [PATCH 40/46] fix: adjust the the varData offset of tp row --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 3387d72a5d..0df371b599 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1074,6 +1074,7 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch } else { ASSERT(pTColumn->type == pColData->type); + SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; if (pRow->statis) { @@ -1084,7 +1085,7 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char*)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { @@ -1106,7 +1107,7 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = pRow->data + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); + void *pData = (char*)pRow + *(int32_t *)(pRow->data + pTColumn->offset - sizeof(TSKEY)); cv.value.nData = varDataLen(pData); cv.value.pData = varDataVal(pData); } else { @@ -1149,7 +1150,6 @@ static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSch if (code) goto _exit; } else { ASSERT(pTColumn->type == pColData->type); - ASSERT(pTColumn->colId == pColData->cid); SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; TDRowValT vt = TD_VTYPE_NONE; // default is NONE From 08aaf9765785b4c3cde52a660afc3926a7715de8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 19:11:06 +0800 Subject: [PATCH 41/46] chore: code optimization --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 0df371b599..755a551e20 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1074,7 +1074,6 @@ static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSch } else { ASSERT(pTColumn->type == pColData->type); - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; if (pRow->statis) { From 52b28fbceb3b7a9e9a88eb7bc8a40009d27968c1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Nov 2022 19:16:51 +0800 Subject: [PATCH 42/46] fix ahandle mem leak --- source/client/src/clientHb.c | 5 ++++- source/libs/transport/inc/transComm.h | 2 +- source/libs/transport/src/transCli.c | 28 ++++++++++++++++++++++++--- source/libs/transport/src/transComm.c | 2 +- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index bf34d3e2df..0f881beb66 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -61,7 +61,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); - tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, rsp->uid); + tscDebug("hb db rsp, db:%s, vgVersion:%d, stateTs:%" PRId64 ", uid:%" PRIx64, rsp->db, rsp->vgVersion, rsp->stateTs, + rsp->uid); if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); @@ -293,6 +294,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); tFreeClientHbBatchRsp(&pRsp); return -1; } @@ -322,6 +324,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { tFreeClientHbBatchRsp(&pRsp); taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); return code; } diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index f33067f62a..ac54749ae1 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -96,7 +96,7 @@ typedef void* queue[2]; //#define TRANS_RETRY_COUNT_LIMIT 100 // retry count limit //#define TRANS_RETRY_INTERVAL 15 // retry interval (ms) -#define TRANS_CONN_TIMEOUT 3000 // connect timeout (s) +#define TRANS_CONN_TIMEOUT 3000 // connect timeout (ms) #define TRANS_READ_TIMEOUT 3000 // read timeout (ms) #define TRANS_PACKET_LIMIT 1024 * 1024 * 512 diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 700c5ee1c9..a0df371bb6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -80,6 +80,7 @@ typedef struct SCliThrd { uint64_t nextTimeout; // next timeout void* pTransInst; // + void (*destroyAhandleFp)(void* ahandle); SHashObj* fqdn2ipCache; SCvtAddr cvtAddr; @@ -158,6 +159,7 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, static FORCE_INLINE void destroyUserdata(STransMsg* userdata); static FORCE_INLINE void destroyCmsg(void* cmsg); +static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg); static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); @@ -569,7 +571,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { if (conn->list->size >= 50) { STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); arg->param1 = conn; - arg->param2 = NULL; + arg->param2 = thrd; STrans* pTransInst = thrd->pTransInst; conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); @@ -693,8 +695,8 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } if (conn->timer != NULL) { uv_timer_stop(conn->timer); - taosArrayPush(pThrd->timerList, &conn->timer); conn->timer->data = NULL; + taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; } @@ -1213,6 +1215,25 @@ static FORCE_INLINE void destroyCmsg(void* arg) { taosMemoryFree(pMsg); } +static FORCE_INLINE void destroyCmsgAndAhandle(void* param) { + if (param == NULL) return; + + STaskArg* arg = param; + SCliMsg* pMsg = arg->param1; + SCliThrd* pThrd = arg->param2; + + tDebug("destroy Ahandle A"); + if (pThrd != NULL && pThrd->destroyAhandleFp != NULL) { + tDebug("destroy Ahandle B"); + pThrd->destroyAhandleFp(pMsg->ctx->ahandle); + } + tDebug("destroy Ahandle C"); + + transDestroyConnCtx(pMsg->ctx); + destroyUserdata(&pMsg->msg); + taosMemoryFree(pMsg); +} + static SCliThrd* createThrdObj(void* trans) { STrans* pTransInst = trans; @@ -1247,6 +1268,7 @@ static SCliThrd* createThrdObj(void* trans) { pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = trans; + pThrd->destroyAhandleFp = pTransInst->destroyFp; pThrd->fqdn2ipCache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pThrd->quit = false; return pThrd; @@ -1262,7 +1284,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg); transAsyncPoolDestroy(pThrd->asyncPool); - transDQDestroy(pThrd->delayQueue, destroyCmsg); + transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle); transDQDestroy(pThrd->timeoutQueue, NULL); tDebug("thread destroy %" PRId64, pThrd->pid); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 18b812f314..7710abcaa1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -497,7 +497,7 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { SDelayTask* task = container_of(minNode, SDelayTask, node); STaskArg* arg = task->arg; - if (freeFunc) freeFunc(arg->param1); + if (freeFunc) freeFunc(arg); taosMemoryFree(arg); taosMemoryFree(task); From 197d5d4b7ad558698c708927bf1cdc6f94de0c5c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Nov 2022 19:41:02 +0800 Subject: [PATCH 43/46] free unfinished ahandle while quit --- source/libs/transport/src/transCli.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a0df371bb6..73220517ab 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -263,16 +263,15 @@ static void* cliWorkThread(void* arg); static void cliReleaseUnfinishedMsg(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { conn->ctx.freeFunc(msg->ctx->ahandle); - } else if (msg->ctx->ahandle != NULL && pTransInst->destroyFp != NULL) { + } else if (msg->ctx->ahandle != NULL && pThrd->destroyAhandleFp != NULL) { tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle); - pTransInst->destroyFp(msg->ctx->ahandle); + pThrd->destroyAhandleFp(msg->ctx->ahandle); } } destroyCmsg(msg); From 174ba6ad6e350955398d707842696faf0df3a898 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Nov 2022 21:18:36 +0800 Subject: [PATCH 44/46] conn timeout refactor --- source/libs/transport/src/transCli.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 73220517ab..f682bc19a3 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -846,10 +846,12 @@ void cliConnCb(uv_connect_t* req, int status) { SCliConn* pConn = req->data; SCliThrd* pThrd = pConn->hostThrd; - uv_timer_stop(pConn->timer); - pConn->timer->data = NULL; - taosArrayPush(pThrd->timerList, &pConn->timer); - pConn->timer = NULL; + if (pConn->timer != NULL) { + uv_timer_stop(pConn->timer); + pConn->timer->data = NULL; + taosArrayPush(pThrd->timerList, &pConn->timer); + pConn->timer = NULL; + } if (status != 0) { tError("%s conn %p failed to connect server:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_strerror(status)); From 5a0da6d28d00101e701f4145be96e869df2f43bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 11 Nov 2022 21:42:23 +0800 Subject: [PATCH 45/46] conn timeout refactor --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f682bc19a3..4fb00b1a6d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -477,7 +477,7 @@ void cliConnTimeout(uv_timer_t* handle) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; - cliHandleExceptImpl(conn, TSDB_CODE_RPC_TIMEOUT); + cliHandleExceptImpl(conn, -1); } void cliReadTimeoutCb(uv_timer_t* handle) { // set up timeout cb From 33861a9d4c8eba4b3849a0378f39b14816af91b6 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 11 Nov 2022 22:13:21 +0800 Subject: [PATCH 46/46] fix: allocate memory for lock --- source/dnode/vnode/src/vnd/vnodeBufPool.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 2fa88badf6..71e926bd35 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -28,7 +28,14 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) } if (VND_IS_RSMA(pVnode)) { + pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); + if (!pPool->lock) { + taosMemoryFree(pPool); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } if (taosThreadSpinInit(pPool->lock, 0) != 0) { + taosMemoryFree((void*)pPool->lock); taosMemoryFree(pPool); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -53,7 +60,10 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) static int vnodeBufPoolDestroy(SVBufPool *pPool) { vnodeBufPoolReset(pPool); - if (pPool->lock) taosThreadSpinDestroy(pPool->lock); + if (pPool->lock) { + taosThreadSpinDestroy(pPool->lock); + taosMemoryFree((void*)pPool->lock); + } taosMemoryFree(pPool); return 0; }