From a999e56a1dad1dd2ead3c6288cf8ec223990d984 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 5 Feb 2023 16:08:34 +0800 Subject: [PATCH 01/28] typo: pthread_cleanup_push/pop better exist in pair --- source/os/src/osTimer.c | 37 ++++++++++++++++++----------------- tools/shell/src/shellEngine.c | 32 ++++++++++++++++-------------- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index d1c233ea9c..185bd9832a 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -114,27 +114,28 @@ static void *taosProcessAlarmSignal(void *tharg) { taosThreadCleanupPush(taosDeleteTimer, &timerId); - struct itimerspec ts; - ts.it_value.tv_sec = 0; - ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK; - ts.it_interval.tv_sec = 0; - ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; + do { + struct itimerspec ts; + ts.it_value.tv_sec = 0; + ts.it_value.tv_nsec = 1000000 * MSECONDS_PER_TICK; + ts.it_interval.tv_sec = 0; + ts.it_interval.tv_nsec = 1000000 * MSECONDS_PER_TICK; - if (timer_settime(timerId, 0, &ts, NULL)) { - // printf("Failed to init timer"); - return NULL; - } - - int signo; - while (!stopTimer) { - if (sigwait(&sigset, &signo)) { - // printf("Failed to wait signal: number %d", signo); - continue; + if (timer_settime(timerId, 0, &ts, NULL)) { + // printf("Failed to init timer"); + break; } - /* //printf("Signal handling: number %d ......\n", signo); */ - callback(0); - } + int signo; + while (!stopTimer) { + if (sigwait(&sigset, &signo)) { + // printf("Failed to wait signal: number %d", signo); + continue; + } + /* //printf("Signal handling: number %d ......\n", signo); */ + callback(0); + } + } while (0); taosThreadCleanupPop(1); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 479c2cf39a..d45c4fbdb5 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1046,26 +1046,28 @@ void *shellThreadLoop(void *arg) { taosGetOldTerminalMode(); taosThreadCleanupPush(shellCleanup, NULL); - char *command = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); - if (command == NULL) { - printf("failed to malloc command\r\n"); - return NULL; - } - do { - memset(command, 0, SHELL_MAX_COMMAND_SIZE); - taosSetTerminalMode(); - - if (shellReadCommand(command) != 0) { + char *command = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + if (command == NULL) { + printf("failed to malloc command\r\n"); break; } - taosResetTerminalMode(); - } while (shellRunCommand(command, true) == 0); + do { + memset(command, 0, SHELL_MAX_COMMAND_SIZE); + taosSetTerminalMode(); - taosMemoryFreeClear(command); - shellWriteHistory(); - shellExit(); + if (shellReadCommand(command) != 0) { + break; + } + + taosResetTerminalMode(); + } while (shellRunCommand(command, true) == 0); + + taosMemoryFreeClear(command); + shellWriteHistory(); + shellExit(); + } while (0); taosThreadCleanupPop(1); return NULL; From 8c754bed200dddcc4f796e7790a8351f54e619c3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Feb 2023 14:32:31 +0800 Subject: [PATCH 02/28] fix: mem leak whene fd excced limit --- source/libs/transport/src/transSvr.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index fa8929f7d9..349d679e0d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -679,14 +679,21 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { SServerObj* pObj = container_of(stream, SServerObj, server); uv_tcp_t* cli = (uv_tcp_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); - uv_tcp_init(pObj->loop, cli); + if (cli == NULL) return; - if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + int err = uv_tcp_init(pObj->loop, cli); + if (err != 0) { + tError("failed to create tcp: %s", uv_err_name(err)); + taosMemoryFree(cli); + return; + } + err = uv_accept(stream, (uv_stream_t*)cli); + if (err == 0) { #if defined(WINDOWS) || defined(DARWIN) if (pObj->numOfWorkerReady < pObj->numOfThreads) { tError("worker-threads are not ready for all, need %d instead of %d.", pObj->numOfThreads, pObj->numOfWorkerReady); - uv_close((uv_handle_t*)cli, NULL); + uv_close((uv_handle_t*)cli, uvFreeCb); return; } #endif @@ -702,8 +709,10 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { if (!uv_is_closing((uv_handle_t*)cli)) { - uv_close((uv_handle_t*)cli, NULL); + tError("failed to accept tcp: %s", uv_err_name(err)); + uv_close((uv_handle_t*)cli, uvFreeCb); } else { + tError("failed to accept tcp: %s", uv_err_name(err)); taosMemoryFree(cli); } } From 181d588f85e5b510bb75ed125e60be62aacf014e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 24 Feb 2023 16:04:35 +0800 Subject: [PATCH 03/28] fix:core dump if data block size is 0 --- source/dnode/vnode/src/tq/tqExec.c | 2 +- tests/system-test/7-tmq/subscribeDb4.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 093186ebbb..97612d3582 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -152,7 +152,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } tqDebug("tmqsnap task execute end, get %p", pDataBlock); - if (pDataBlock != NULL) { + if (pDataBlock != NULL && pDataBlock->info.rows > 0) { if (pRsp->withTbName) { int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 7f5169361c..1395359a1f 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -26,7 +26,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 10000, + 'rowsPerTbl': 50000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 20, From 5830e0931ca43d8a63919f7d00da19f41e6a9c0c Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Mon, 27 Feb 2023 15:22:46 +0800 Subject: [PATCH 04/28] test:add test case for TD-22532 --- tests/system-test/0-others/show.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index b5d6a0d1a3..82f9cb9d7e 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -12,13 +12,16 @@ # -*- coding: utf-8 -*- + + + +import re from util.log import * from util.cases import * from util.sql import * from util.common import * from util.sqlset import * - class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -28,7 +31,7 @@ class TDTestCase: self.ins_param_list = ['dnodes','mnodes','qnodes','cluster','functions','users','grants','topics','subscriptions','streams'] self.perf_param = ['apps','connections','consumers','queries','transactions'] self.perf_param_list = ['apps','connections','consumers','queries','trans'] - + def ins_check(self): tdSql.prepare() for param in self.ins_param_list: @@ -113,7 +116,25 @@ class TDTestCase: query_result = tdSql.queryResult tdSql.checkEqual(query_result[0][1].lower(),sql) tdSql.execute('drop database db') + def check_gitinfo(self): + taosd_gitinfo_sql = '' + tdSql.query('show dnode 1 variables') + for i in tdSql.queryResult: + if i[1].lower() == "gitinfo": + taosd_gitinfo_sql = f"gitinfo: {i[2]}" + taos_gitinfo_sql = '' + tdSql.query('show local variables') + for i in tdSql.queryResult: + if i[0].lower() == "gitinfo": + taos_gitinfo_sql = f"gitinfo: {i[1]}" + taos_info = os.popen('taos -V').read() + taos_gitinfo = re.findall("^gitinfo.*",taos_info,re.M) + tdSql.checkEqual(taos_gitinfo_sql,taos_gitinfo[0]) + taosd_info = os.popen('taosd -V').read() + taosd_gitinfo = re.findall("^gitinfo.*",taosd_info,re.M) + tdSql.checkEqual(taosd_gitinfo_sql,taosd_gitinfo[0]) def run(self): + self.check_gitinfo() self.ins_check() self.perf_check() self.show_sql() From 031e9032d0c7e25d62d6bd6d4f1fe52ce525d900 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 27 Feb 2023 18:59:38 +0800 Subject: [PATCH 05/28] fix:asan error --- source/dnode/mnode/impl/src/mndStream.c | 2 +- tests/script/tsim/stream/basic1.sim | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 2a05511134..b7f80f6b0e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1223,7 +1223,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // node id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - int32_t nodeId = TMAX(pTask->nodeId, 0); + int64_t nodeId = TMAX(pTask->nodeId, 0); colDataSetVal(pColInfo, numOfRows, (const char *)&nodeId, false); // level diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index c61c7667f8..a8f04aac64 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -887,4 +887,20 @@ if $rows != 1 then goto loop19 endi +print select * from information_schema.ins_stream_tasks; +sql select * from information_schema.ins_stream_tasks; + +if $rows == 0 then + print =====rows=$rows + return -1 +endi + +print select * from information_schema.ins_streams; +sql select * from information_schema.ins_streams; + +if $rows == 0 then + print =====rows=$rows + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2b495e6c3314a81fc71d487f29f0fdd2bc165bb7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Mar 2023 10:53:01 +0800 Subject: [PATCH 06/28] fix:reduce the amount of insert data to avoid CI timeout --- tests/system-test/7-tmq/subscribeDb4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index d835603678..c14d3b27b1 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -26,7 +26,7 @@ class TDTestCase: 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, 'ctbNum': 10, - 'rowsPerTbl': 50000, + 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 20, From d7cbcbf8ef0ce09025c3c518a77177056209d355 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Mar 2023 10:55:55 +0800 Subject: [PATCH 07/28] fix:open test cases --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index c70a50867b..67d488444d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1086,7 +1086,7 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R From ee70606f19e2d0bb2e94d54a123c277a2f170b91 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 1 Mar 2023 14:48:15 +0800 Subject: [PATCH 08/28] fix: projection query in descending order result error issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +- tests/script/api/insertSameTs.c | 136 +++++++++++++++++++++++++ tests/script/api/makefile | 1 + 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 tests/script/api/insertSameTs.c diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a158bbfa8d..4c5b87559e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3512,6 +3512,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn CHECK_FILEBLOCK_STATE* state) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; + bool asc = ASCENDING_TRAVERSE(pReader->order); *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; @@ -3522,7 +3523,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn if (loadNeighbor && (code == TSDB_CODE_SUCCESS)) { pDumpInfo->rowIndex = doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, key, pMerger, &pReader->verRange, step); - if (pDumpInfo->rowIndex >= pDumpInfo->totalRows) { + if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) { *state = CHECK_FILEBLOCK_CONT; } } diff --git a/tests/script/api/insertSameTs.c b/tests/script/api/insertSameTs.c new file mode 100644 index 0000000000..ff5e4dd154 --- /dev/null +++ b/tests/script/api/insertSameTs.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +#include +#include +#include +#include +#include "taos.h" // TAOS header file + +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + +void Test(TAOS *taos, char *qstr); + +int main(int argc, char *argv[]) { + char qstr[1024]; + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/); + exit(1); + } + Test(taos, qstr); + taos_close(taos); + taos_cleanup(); +} +void Test(TAOS *taos, char *qstr) { + queryDB(taos, "drop database if exists demo"); + queryDB(taos, "create database demo vgroups 1 minrows 10"); + TAOS_RES *result; + queryDB(taos, "use demo"); + + queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))"); + printf("success to create table\n"); + + int i = 0; + for (int32_t n = 0; n < 10; ++n) { + for (i = 0; i < 10; ++i) { + int32_t v = n * 10 + i; + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)1546300800000, v, v, v, v*10000000, v*1.0, v*2.0, "hello"); + printf("qstr: %s\n", qstr); + + TAOS_RES *result1 = taos_query(taos, qstr); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to insert row, reason:%s\n", taos_errstr(result1)); + taos_free_result(result1); + exit(1); + } else { + printf("insert row: %i\n", v); + } + taos_free_result(result1); + + strcpy(qstr, "flush database demo"); + result1 = taos_query(taos, qstr); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to fluash database, reason:%s\n", taos_errstr(result1)); + taos_free_result(result1); + exit(1); + } + taos_free_result(result1); + } + } + + // query the records + sprintf(qstr, "SELECT * FROM m1 order by ts desc"); + result = taos_query(taos, qstr); + if (result == NULL || taos_errno(result) != 0) { + printf("failed to select, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_field_count(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + + printf("num_fields = %d\n", num_fields); + printf("select * from table order by ts desc, result:\n"); + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + char temp[1024] = {0}; + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); +} + diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 75a2273f12..6739794cc8 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -14,6 +14,7 @@ exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) + gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) clean: rm $(ROOT)batchprepare From 9ad236b8f7278355c553927b8ca801ad48a03f36 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Mar 2023 17:03:52 +0800 Subject: [PATCH 09/28] fix:disable test cases --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 67d488444d..c70a50867b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1086,7 +1086,7 @@ ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py -,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +#,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R From 8a23f0ce3bc291e6ace9f458e6711592d102b7d2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Wed, 1 Mar 2023 18:59:27 +0800 Subject: [PATCH 10/28] fix:set timestamp precision --- source/common/src/tdatablock.c | 2 +- source/libs/parser/src/parTranslater.c | 3 ++- source/libs/scalar/src/sclvector.c | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 88eb3bdb97..453e4a8d8e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2003,7 +2003,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); - formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + formatTimestamp(pBuf, *(uint64_t*)var, pColInfoData->info.precision); len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); if (len >= size - 1) return dumpBuf; break; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2f621ba5c8..79dd658987 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1211,7 +1211,8 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD } else { res = translateNormalValue(pCxt, pVal, targetDt, strict); } - pVal->node.resType = targetDt; + pVal->node.resType.type = targetDt.type; + pVal->node.resType.bytes = targetDt.bytes; pVal->node.resType.scale = pVal->unit; pVal->translate = true; if (!strict && TSDB_DATA_TYPE_UBIGINT == pVal->node.resType.type && pVal->datum.u <= INT64_MAX) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 3fe016d444..4d803cb638 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -915,6 +915,7 @@ int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_ SDataType t = {.type = type}; t.bytes = IS_VAR_DATA_TYPE(t.type)? input->columnData->info.bytes:tDataTypes[type].bytes; + t.precision = input->columnData->info.precision; int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output); if (code != TSDB_CODE_SUCCESS) { From 8f858268d396a89819384ee175f91e78707c065f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 1 Mar 2023 19:25:24 +0800 Subject: [PATCH 11/28] fix: add test case --- tests/parallel_test/cases.task | 6 +++ tests/system-test/2-query/projectionDesc.py | 47 +++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tests/system-test/2-query/projectionDesc.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 93e11eb1fb..ef70c6e8da 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -632,6 +632,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py @@ -655,6 +657,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py ,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 @@ -854,6 +857,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 @@ -952,6 +956,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 @@ -1070,6 +1075,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py diff --git a/tests/system-test/2-query/projectionDesc.py b/tests/system-test/2-query/projectionDesc.py new file mode 100644 index 0000000000..6bb9aff4f3 --- /dev/null +++ b/tests/system-test/2-query/projectionDesc.py @@ -0,0 +1,47 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.batchNum = 5 + self.ts = 1537146000000 + + def run(self): + dbname = "db" + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') + tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") + for n in range(self.batchNum): + for i in range(self.rowNum): + tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + tdSql.execute(f"flush database {dbname}") + + tdSql.query(f"select * from {dbname}.stb_1 order by ts desc") + tdSql.checkRows(1) + #tdSql.checkData(0,0,1537146000000) + tdSql.checkData(0,1,10) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 37348fb45639ed933f6af127e7c4258e7e5c56ae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Mar 2023 21:00:01 +0800 Subject: [PATCH 12/28] change default config --- source/common/src/tglobal.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 99795fcc79..b48bce78fc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -521,7 +521,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem = cfgGetItem(tsCfg, "numOfRpcSessions"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfRpcSessions = 2000; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000); pItem->i32 = tsNumOfRpcSessions; pItem->stype = stype; @@ -529,7 +528,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem = cfgGetItem(tsCfg, "timeToGetAvailableConn"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsTimeToGetAvailableConn = 1000; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000); pItem->i32 = tsTimeToGetAvailableConn; pItem->stype = stype; From 8cdae7933b9795183b05b3962c86b97274fd657a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Mar 2023 21:03:24 +0800 Subject: [PATCH 13/28] change default config --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index b48bce78fc..5e28a8a4c0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -42,7 +42,7 @@ bool tsPrintAuth = false; // queue & threads int32_t tsNumOfRpcThreads = 1; int32_t tsNumOfRpcSessions = 5000; -int32_t tsTimeToGetAvailableConn = 100000; +int32_t tsTimeToGetAvailableConn = 2000000; int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 4; int32_t tsNumOfMnodeQueryThreads = 4; From 4fa1839798b299f2b004121cda3ea8c4beb62bfe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Mar 2023 09:33:46 +0800 Subject: [PATCH 14/28] change default config --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5e28a8a4c0..eef174ef5f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -42,7 +42,7 @@ bool tsPrintAuth = false; // queue & threads int32_t tsNumOfRpcThreads = 1; int32_t tsNumOfRpcSessions = 5000; -int32_t tsTimeToGetAvailableConn = 2000000; +int32_t tsTimeToGetAvailableConn = 1000000; int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 4; int32_t tsNumOfMnodeQueryThreads = 4; From a484725f314299dd655cdd91d7f24d0d8e914036 Mon Sep 17 00:00:00 2001 From: Hui Li <52318143+plum-lihui@users.noreply.github.com> Date: Thu, 2 Mar 2023 10:56:09 +0800 Subject: [PATCH 15/28] Update cases.task --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 93e11eb1fb..5204ee1fb1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -738,7 +738,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py From 2a5074870db6196ba5dcee0eddce4c027e53f5e2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 2 Mar 2023 11:33:29 +0800 Subject: [PATCH 16/28] fix: add missing db options in show create database result --- include/common/tmsg.h | 7 +++++++ include/libs/nodes/cmdnodes.h | 1 + source/common/src/tmsg.c | 14 ++++++++++++++ source/dnode/mnode/impl/src/mndDb.c | 7 +++++++ source/libs/command/src/command.c | 16 ++++++++++------ source/libs/parser/src/parTranslater.c | 5 +++++ 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6ca36e7433..41bc873a3b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -929,12 +929,19 @@ typedef struct { int32_t minRows; int32_t maxRows; int32_t walFsyncPeriod; + int16_t hashPrefix; + int16_t hashSuffix; int8_t walLevel; int8_t precision; int8_t compression; int8_t replications; int8_t strict; int8_t cacheLast; + int32_t tsdbPageSize; + int32_t walRetentionPeriod; + int32_t walRollPeriod; + int64_t walRetentionSize; + int64_t walSegmentSize; int32_t numOfRetensions; SArray* pRetensions; int8_t schemaless; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 164fbf018c..a0fc8d1238 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -265,6 +265,7 @@ typedef struct SShowStmt { typedef struct SShowCreateDatabaseStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; void* pCfg; // SDbCfgInfo } SShowCreateDatabaseStmt; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e180959d1e..d8e987e879 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2865,12 +2865,19 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->minRows) < 0) return -1; if (tEncodeI32(&encoder, pRsp->maxRows) < 0) return -1; if (tEncodeI32(&encoder, pRsp->walFsyncPeriod) < 0) return -1; + if (tEncodeI16(&encoder, pRsp->hashPrefix) < 0) return -1; + if (tEncodeI16(&encoder, pRsp->hashSuffix) < 0) return -1; if (tEncodeI8(&encoder, pRsp->walLevel) < 0) return -1; if (tEncodeI8(&encoder, pRsp->precision) < 0) return -1; if (tEncodeI8(&encoder, pRsp->compression) < 0) return -1; if (tEncodeI8(&encoder, pRsp->replications) < 0) return -1; if (tEncodeI8(&encoder, pRsp->strict) < 0) return -1; if (tEncodeI8(&encoder, pRsp->cacheLast) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->tsdbPageSize) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->walRetentionPeriod) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->walRollPeriod) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->walRetentionSize) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->walSegmentSize) < 0) return -1; if (tEncodeI32(&encoder, pRsp->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pRsp->numOfRetensions; ++i) { SRetention *pRetension = taosArrayGet(pRsp->pRetensions, i); @@ -2905,12 +2912,19 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { if (tDecodeI32(&decoder, &pRsp->minRows) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->maxRows) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->walFsyncPeriod) < 0) return -1; + if (tDecodeI16(&decoder, &pRsp->hashPrefix) < 0) return -1; + if (tDecodeI16(&decoder, &pRsp->hashSuffix) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->walLevel) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->precision) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->compression) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->replications) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->strict) < 0) return -1; if (tDecodeI8(&decoder, &pRsp->cacheLast) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->tsdbPageSize) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->walRetentionPeriod) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->walRollPeriod) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->walRetentionSize) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->walSegmentSize) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->numOfRetensions) < 0) return -1; if (pRsp->numOfRetensions > 0) { pRsp->pRetensions = taosArrayInit(pRsp->numOfRetensions, sizeof(SRetention)); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5de76a9d46..af76971304 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -887,12 +887,19 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { cfgRsp.minRows = pDb->cfg.minRows; cfgRsp.maxRows = pDb->cfg.maxRows; cfgRsp.walFsyncPeriod = pDb->cfg.walFsyncPeriod; + cfgRsp.hashPrefix = pDb->cfg.hashPrefix; + cfgRsp.hashSuffix = pDb->cfg.hashSuffix; cfgRsp.walLevel = pDb->cfg.walLevel; cfgRsp.precision = pDb->cfg.precision; cfgRsp.compression = pDb->cfg.compression; cfgRsp.replications = pDb->cfg.replications; cfgRsp.strict = pDb->cfg.strict; cfgRsp.cacheLast = pDb->cfg.cacheLast; + cfgRsp.tsdbPageSize = pDb->cfg.tsdbPageSize; + cfgRsp.walRetentionPeriod = pDb->cfg.walRetentionPeriod; + cfgRsp.walRollPeriod = pDb->cfg.walRollPeriod; + cfgRsp.walRetentionSize = pDb->cfg.walRetentionSize; + cfgRsp.walSegmentSize = pDb->cfg.walSegmentSize; cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; cfgRsp.pRetensions = pDb->cfg.pRetensions; cfgRsp.schemaless = pDb->cfg.schemaless; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index f88701afe2..c862a75ed3 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -248,13 +248,13 @@ static const char* cacheModelStr(int8_t cacheModel) { return TSDB_CACHE_MODEL_NONE_STR; } -static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) { +static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { blockDataEnsureCapacity(pBlock, 1); pBlock->info.rows = 1; SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); char buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0}; - STR_TO_VARSTR(buf1, dbFName); + STR_TO_VARSTR(buf1, dbName); colDataSetVal(pCol1, 0, buf1, false); SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); @@ -277,16 +277,20 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S } char* retentions = buildRetension(pCfg->pRetensions); + int32_t dbFNameLen = strlen(dbFName); + int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0; len += sprintf( buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " - "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d", - dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, + "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " + "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 " WAL_ROLL_PERIOD %d WAL_SEGMENT_SIZE %" PRId64, + dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, - 1 == pCfg->numOfStables); + 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, + pCfg->walRetentionSize, pCfg->walRollPeriod, pCfg->walSegmentSize); if (retentions) { len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); @@ -404,7 +408,7 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT SSDataBlock* pBlock = NULL; int32_t code = buildCreateDBResultDataBlock(&pBlock); if (TSDB_CODE_SUCCESS == code) { - setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); + setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->dbFName, pStmt->pCfg); } if (TSDB_CODE_SUCCESS == code) { code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 2f621ba5c8..a20fe9a8b8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6477,6 +6477,11 @@ static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateD if (NULL == pStmt->pCfg) { return TSDB_CODE_OUT_OF_MEMORY; } + + SName name; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + tNameGetFullDbName(&name, pStmt->dbFName); + return getDBCfg(pCxt, pStmt->dbName, (SDbCfgInfo*)pStmt->pCfg); } From 5c50eeb5cccbb266c9b4ceceef0d256a7a50ea10 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Mar 2023 12:16:28 +0800 Subject: [PATCH 17/28] change default config --- source/libs/transport/src/transCli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 526b4bfc6f..274d153adc 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1173,7 +1173,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { addr.sin_port = (uint16_t)htons(pList->port); tTrace("%s conn %p try to connect to %s", pTransInst->label, conn, pList->dst); - int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 4); + int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT * 10); if (fd == -1) { tError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(TAOS_SYSTEM_ERROR(errno))); From 1cb6e49c79e31dce3a54c80d5d978007ee4a5198 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Mar 2023 13:04:49 +0800 Subject: [PATCH 18/28] change default config --- source/libs/stream/src/streamDispatch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index f2b1db19e8..7e7c23f98a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -278,6 +278,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; msg.msgType = TDMT_STREAM_RECOVER_FINISH; + msg.info.noResp = 1; tmsgSendReq(pEpSet, &msg); @@ -522,4 +523,3 @@ FREE: taosFreeQitem(pBlock); return code; } - From 3015967484814513ce603fed6a1f2dcc723393ce Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 2 Mar 2023 13:09:50 +0800 Subject: [PATCH 19/28] fix: fix show create database issue --- tests/develop-test/2-query/show_create_db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py index e5a79074ef..5574a59ec2 100644 --- a/tests/develop-test/2-query/show_create_db.py +++ b/tests/develop-test/2-query/show_create_db.py @@ -42,17 +42,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") self.restartTaosd(1, dbname='scd') @@ -60,17 +60,17 @@ class TDTestCase: tdSql.query('show create database scd;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd') - tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") tdSql.query('show create database scd2;') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd2') - tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") tdSql.query('show create database scd4') tdSql.checkRows(1) tdSql.checkData(0, 0, 'scd4') - tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0") + tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 0 WAL_RETENTION_SIZE 0 WAL_ROLL_PERIOD 0 WAL_SEGMENT_SIZE 0") tdSql.execute('drop database scd') From e383afc96de869dd19e0fef96301f19b99f98361 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Mar 2023 13:58:03 +0800 Subject: [PATCH 20/28] change default config --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index eef174ef5f..e7873fb269 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -41,8 +41,8 @@ bool tsPrintAuth = false; // queue & threads int32_t tsNumOfRpcThreads = 1; -int32_t tsNumOfRpcSessions = 5000; -int32_t tsTimeToGetAvailableConn = 1000000; +int32_t tsNumOfRpcSessions = 6000; +int32_t tsTimeToGetAvailableConn = 500000; int32_t tsNumOfCommitThreads = 2; int32_t tsNumOfTaskQueueThreads = 4; int32_t tsNumOfMnodeQueryThreads = 4; From 09ac1be5ffd3b5b2632c6c2adb31a08978dc6aaa Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 2 Mar 2023 14:26:11 +0800 Subject: [PATCH 21/28] ci:adjust test cases --- tests/script/tsim/stream/basic0.sim | 62 ++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 8e3e99978b..68c5894cbf 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -48,24 +48,35 @@ sleep 100 #=================================================================== print =============== query data from child table +$loop_count = 0 + +loop0: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + sql select `_wstart`,`min(k)`,`max(k)`,sum_alias from outstb + print rows: $rows print $data00 $data01 $data02 $data03 if $rows != 1 then - return -1 + goto loop0 endi if $data01 != 234 then - return -1 + goto loop0 endi if $data02 != 234 then - return -1 + goto loop0 endi if $data03 != 234 then print expect 234, actual $data03 - return -1 + goto loop0 endi #=================================================================== @@ -77,23 +88,34 @@ sleep 100 #=================================================================== print =============== query data from child table +$loop_count = 0 + +loop1: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + sql select `_wstart`,`min(k)`,`max(k)`,sum_alias from outstb + print rows: $rows print $data00 $data01 $data02 $data03 if $rows != 1 then - return -1 + goto loop1 endi if $data01 != -111 then - return -1 + goto loop1 endi if $data02 != 234 then - return -1 + goto loop1 endi if $data03 != 123 then - return -1 + goto loop1 endi #=================================================================== @@ -105,36 +127,46 @@ sleep 100 #=================================================================== print =============== query data from child table +$loop_count = 0 + +loop2: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + sql select `_wstart`,`min(k)`,`max(k)`,sum_alias from outstb print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 if $rows != 2 then - return -1 + goto loop2 endi if $data01 != -111 then - return -1 + goto loop2 endi if $data02 != 234 then - return -1 + goto loop2 endi if $data03 != 123 then - return -1 + goto loop2 endi if $data11 != 789 then - return -1 + goto loop2 endi if $data12 != 789 then - return -1 + goto loop2 endi if $data13 != 789 then - return -1 + goto loop2 endi _OVER: From e0819db5de2d278316f3424d06c5f3089aa8ef81 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 2 Mar 2023 15:06:11 +0800 Subject: [PATCH 22/28] fix: auto retention --- source/dnode/vnode/src/vnd/vnodeSvr.c | 28 ++++++--------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3525e696c5..5712a8ecb8 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -599,16 +599,10 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); -// process -#if 0 - code = tsdbDoRetention(pVnode->pTsdb, trimReq.timestamp); - if (code) goto _exit; - - code = smaDoRetention(pVnode->pSma, trimReq.timestamp); - if (code) goto _exit; -#else + // process vnodeAsyncRentention(pVnode, trimReq.timestamp); -#endif + tsem_wait(&pVnode->canCommit); + tsem_post(&pVnode->canCommit); _exit: return code; @@ -633,18 +627,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p tqUpdateTbUidList(pVnode->pTq, tbUids, false); } -#if 0 - // process - ret = tsdbDoRetention(pVnode->pTsdb, ttlReq.timestamp); - if (ret) goto end; - - ret = smaDoRetention(pVnode->pSma, ttlReq.timestamp); - if (ret) goto end; -#else vnodeAsyncRentention(pVnode, ttlReq.timestamp); - tsem_wait(&pVnode->canCommit); - tsem_post(&pVnode->canCommit); -#endif end: taosArrayDestroy(tbUids); @@ -1647,7 +1630,8 @@ static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void return vnodeProcessCompactVnodeReqImpl(pVnode, version, pReq, len, pRsp); } - #ifndef TD_ENTERPRISE -int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } +int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + return 0; +} #endif From 7deee1c4c78d8ce9ae51c5475f466a5f6e2c7983 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 2 Mar 2023 18:22:42 +0800 Subject: [PATCH 23/28] fix: last cache read problem after schema change --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCache.c | 93 ++++++++++++++++----- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 4 +- 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 44b7e95f9e..15087e90f0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -766,6 +766,7 @@ typedef struct SCacheRowsReader { TdThreadMutex readerMutex; SVnode *pVnode; STSchema *pSchema; + STSchema *pCurrSchema; uint64_t uid; uint64_t suid; char **transferBuf; // todo remove it soon diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 79e0ccc3e3..f69445fa0c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1390,17 +1390,56 @@ _err: return code; } -static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) { - int32_t code = 0; +static int32_t cloneTSchema(STSchema *pSrc, STSchema **ppDst) { + int32_t len = sizeof(STSchema) + sizeof(STColumn) * pSrc->numOfCols; + *ppDst = taosMemoryMalloc(len); + if (NULL == *ppDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(*ppDst, pSrc, len); + return TSDB_CODE_SUCCESS; +} +static int32_t updateTSchema(int32_t sversion, SCacheRowsReader *pReader, uint64_t uid) { + if (NULL == pReader->pCurrSchema && sversion == pReader->pSchema->version) { + return cloneTSchema(pReader->pSchema, &pReader->pCurrSchema); + } + + if (NULL != pReader->pCurrSchema && sversion == pReader->pCurrSchema->version) { + return TSDB_CODE_SUCCESS; + } + + taosMemoryFreeClear(pReader->pCurrSchema); + return metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pCurrSchema); +} + +static int32_t initLastColArray(STSchema *pTSchema, SArray **ppColArray) { + SArray *pColArray = taosArrayInit(pTSchema->numOfCols, sizeof(SLastCol)); + if (NULL == pColArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pTSchema->numOfCols; ++i) { + SLastCol col = {.ts = 0, .colVal = COL_VAL_NULL(pTSchema->columns[i].colId, pTSchema->columns[i].type)}; + taosArrayPush(pColArray, &col); + } + *ppColArray = pColArray; + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr) { STSchema *pTSchema = pr->pSchema; // metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - int16_t nCol = pTSchema->numOfCols; - int16_t iCol = 0; + int16_t nLastCol = pTSchema->numOfCols; int16_t noneCol = 0; bool setNoneCol = false; - SArray *pColArray = taosArrayInit(nCol, sizeof(SLastCol)); + SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; + int32_t code = initLastColArray(pTSchema, &pColArray); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; @@ -1415,6 +1454,13 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach break; } + code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; + int16_t nCol = pTSchema->numOfCols; + TSKEY rowTs = TSDBROW_TS(pRow); if (lastRowTs == TSKEY_MAX) { @@ -1422,28 +1468,27 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach STColumn *pTColumn = &pTSchema->columns[0]; *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); - if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + taosArraySet(pColArray, 0, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}); - for (iCol = 1; iCol < nCol; ++iCol) { + for (int16_t iCol = 1; iCol < nCol; ++iCol) { + if (iCol >= nLastCol) { + break; + } + SLastCol *pCol = taosArrayGet(pColArray, iCol); + if (pCol->colVal.cid != pTSchema->columns[iCol].colId) { + continue; + } tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - SLastCol lastCol = {.ts = lastRowTs, .colVal = *pColVal}; + *pCol = (SLastCol){.ts = lastRowTs, .colVal = *pColVal}; if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) { - lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { + pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); + if (pCol->colVal.value.pData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData); - } - - if (taosArrayPush(pColArray, &lastCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData); } if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { @@ -1461,10 +1506,16 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach // merge into pColArray setNoneCol = false; - for (iCol = noneCol; iCol < nCol; ++iCol) { + for (int16_t iCol = noneCol; iCol < nCol; ++iCol) { + if (iCol >= nLastCol) { + break; + } // high version's column value SLastCol *lastColVal = (SLastCol *)taosArrayGet(pColArray, iCol); - SColVal *tColVal = &lastColVal->colVal; + if (lastColVal->colVal.cid != pTSchema->columns[iCol].colId) { + continue; + } + SColVal *tColVal = &lastColVal->colVal; tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 97b13a3c37..5bb224799a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -209,6 +209,8 @@ void* tsdbCacherowsReaderClose(void* pReader) { taosMemoryFree(p->pSchema); } + taosMemoryFree(p->pCurrSchema); + destroyLastBlockLoadInfo(p->pLoadInfo); taosMemoryFree((void*)p->idstr); @@ -303,7 +305,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 for (int32_t i = 0; i < pr->pSchema->numOfCols; ++i) { struct STColumn* pCol = &pr->pSchema->columns[i]; - SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type}; + SLastCol p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL}; if (IS_VAR_DATA_TYPE(pCol->type)) { p.colVal.value.pData = taosMemoryCalloc(pCol->bytes, sizeof(char)); From 95660ebed602508db4d4b5558eb63d007919721b Mon Sep 17 00:00:00 2001 From: xinsheng Ren <285808407@qq.com> Date: Thu, 2 Mar 2023 19:31:26 +0800 Subject: [PATCH 24/28] fix: windows release from tdengine/package/release.bat failed (#20232) * fix: windows release from tdengine/package/release.bat failed * fix: TDengine spelling --------- Co-authored-by: facetosea <25808407@qq.com> --- packaging/release.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/release.bat b/packaging/release.bat index 4c82c5ead5..4b49f364d0 100644 --- a/packaging/release.bat +++ b/packaging/release.bat @@ -46,9 +46,9 @@ rd /s /Q C:\TDengine cmake --install . if not %errorlevel% == 0 ( call :RUNFAILED build x64 failed & exit /b 1) cd %package_dir% -iscc /DMyAppInstallName="%packagServerName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release +iscc /DMyAppInstallName="%packagServerName_x64%" /DMyAppVersion="%2" /DCusName="TDengine" /DCusPrompt="taos" /DMyAppExcludeSource="" tools\tdengine.iss /O..\release if not %errorlevel% == 0 ( call :RUNFAILED package %packagServerName_x64% failed & exit /b 1) -iscc /DMyAppInstallName="%packagClientName_x64%" /DMyAppVersion="%2" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release +iscc /DMyAppInstallName="%packagClientName_x64%" /DMyAppVersion="%2" /DCusName="TDengine" /DCusPrompt="taos" /DMyAppExcludeSource="taosd.exe" tools\tdengine.iss /O..\release if not %errorlevel% == 0 ( call :RUNFAILED package %packagClientName_x64% failed & exit /b 1) goto EXIT0 From 7649ede0749734b77b6ab1c2d1897598a7ff04d8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 2 Mar 2023 19:45:21 +0800 Subject: [PATCH 25/28] fix: last cache read problem after schema change --- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f69445fa0c..425bd2ca1c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1432,6 +1432,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach int16_t nLastCol = pTSchema->numOfCols; int16_t noneCol = 0; bool setNoneCol = false; + bool hasRow = false; SArray *pColArray = NULL; SColVal *pColVal = &(SColVal){0}; @@ -1454,6 +1455,8 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach break; } + hasRow = true; + code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); if (TSDB_CODE_SUCCESS != code) { goto _err; @@ -1545,6 +1548,9 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach //*ppLastArray = NULL; // taosArrayDestroy(pColArray); //} else { + if (!hasRow) { + taosArrayClear(pColArray); + } *ppLastArray = pColArray; //} From 302c09cc75580a914a81195d3848bc0ffddcb549 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 2 Mar 2023 21:56:55 +0800 Subject: [PATCH 26/28] ci:add cache in ci compling --- tests/parallel_test/container_build.sh | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index d0086c733e..55df94e610 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -51,10 +51,24 @@ else REP_DIR=/home/TDinternal REP_REAL_PATH=$WORKDIR/TDinternal REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal + fi date docker run \ -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v REP_REAL_PATH/enterprise/src/plugins/taosx/target:REP_DIR/enterprise/src/plugins/taosx/target \ + -v REP_REAL_PATH/community/tools/taosws-rs/target:REP_DIR/community/tools/taosws-rs/target \ + -v REP_REAL_PATH/community/contrib/cJson/:REP_DIR/community/contrib/cJson \ + -v REP_REAL_PATH/community/contrib/googletest/:REP_DIR/community/contrib/googletest \ + -v REP_REAL_PATH/community/contrib/cpp-stub/:REP_DIR/community/contrib/cpp-stub \ + -v REP_REAL_PATH/community/contrib/libuv/:REP_DIR/community/contrib/libuv \ + -v REP_REAL_PATH/community/contrib/lz4/:REP_DIR/community/contrib/lz4 \ + -v REP_REAL_PATH/community/contrib/zlib/:REP_DIR/community/contrib/zlib \ + -v REP_REAL_PATH/community/contrib/jemalloc/:REP_DIR/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true;make -j || exit 1" if [[ -d ${WORKDIR}/debugNoSan ]] ;then @@ -70,6 +84,19 @@ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date docker run \ -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v REP_REAL_PATH/enterprise/src/plugins/taosx/target:REP_DIR/enterprise/src/plugins/taosx/target \ + -v REP_REAL_PATH/community/tools/taosws-rs/target:REP_DIR/community/tools/taosws-rs/target \ + -v REP_REAL_PATH/community/contrib/cJson/:REP_DIR/community/contrib/cJson \ + -v REP_REAL_PATH/community/contrib/googletest/:REP_DIR/community/contrib/googletest \ + -v REP_REAL_PATH/community/contrib/cpp-stub/:REP_DIR/community/contrib/cpp-stub \ + -v REP_REAL_PATH/community/contrib/libuv/:REP_DIR/community/contrib/libuv \ + -v REP_REAL_PATH/community/contrib/lz4/:REP_DIR/community/contrib/lz4 \ + -v REP_REAL_PATH/community/contrib/zlib/:REP_DIR/community/contrib/zlib \ + -v REP_REAL_PATH/community/contrib/jemalloc/:REP_DIR/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan From 72a0eab44add726eb2c0f303c1f84cc1b5b1e2fa Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 2 Mar 2023 22:03:36 +0800 Subject: [PATCH 27/28] ci:add cache in ci compling --- tests/parallel_test/container_build.sh | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 55df94e610..edb9b4ab3c 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -60,15 +60,15 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v REP_REAL_PATH/enterprise/src/plugins/taosx/target:REP_DIR/enterprise/src/plugins/taosx/target \ - -v REP_REAL_PATH/community/tools/taosws-rs/target:REP_DIR/community/tools/taosws-rs/target \ - -v REP_REAL_PATH/community/contrib/cJson/:REP_DIR/community/contrib/cJson \ - -v REP_REAL_PATH/community/contrib/googletest/:REP_DIR/community/contrib/googletest \ - -v REP_REAL_PATH/community/contrib/cpp-stub/:REP_DIR/community/contrib/cpp-stub \ - -v REP_REAL_PATH/community/contrib/libuv/:REP_DIR/community/contrib/libuv \ - -v REP_REAL_PATH/community/contrib/lz4/:REP_DIR/community/contrib/lz4 \ - -v REP_REAL_PATH/community/contrib/zlib/:REP_DIR/community/contrib/zlib \ - -v REP_REAL_PATH/community/contrib/jemalloc/:REP_DIR/community/contrib/jemalloc \ + -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ + -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true;make -j || exit 1" if [[ -d ${WORKDIR}/debugNoSan ]] ;then @@ -88,15 +88,15 @@ docker run \ -v /root/.cargo/git:/root/.cargo/git \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ - -v REP_REAL_PATH/enterprise/src/plugins/taosx/target:REP_DIR/enterprise/src/plugins/taosx/target \ - -v REP_REAL_PATH/community/tools/taosws-rs/target:REP_DIR/community/tools/taosws-rs/target \ - -v REP_REAL_PATH/community/contrib/cJson/:REP_DIR/community/contrib/cJson \ - -v REP_REAL_PATH/community/contrib/googletest/:REP_DIR/community/contrib/googletest \ - -v REP_REAL_PATH/community/contrib/cpp-stub/:REP_DIR/community/contrib/cpp-stub \ - -v REP_REAL_PATH/community/contrib/libuv/:REP_DIR/community/contrib/libuv \ - -v REP_REAL_PATH/community/contrib/lz4/:REP_DIR/community/contrib/lz4 \ - -v REP_REAL_PATH/community/contrib/zlib/:REP_DIR/community/contrib/zlib \ - -v REP_REAL_PATH/community/contrib/jemalloc/:REP_DIR/community/contrib/jemalloc \ + -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ + -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan From d3ab568f7f58e6792d1b07c454f4b24cfc7b0cc9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 3 Mar 2023 00:38:53 +0800 Subject: [PATCH 28/28] ci:cancle "git clean -dx" --- Jenkinsfile2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 916437b914..0177c57538 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -40,7 +40,7 @@ def check_docs() { sh ''' cd ${WKC} git reset --hard - git clean -fxd + git clean -f rm -rf examples/rust/ git remote prune origin git fetch @@ -86,7 +86,7 @@ def pre_test(){ git fetch cd ${WKC} git reset --hard - git clean -fxd + git clean -f rm -rf examples/rust/ git remote prune origin git fetch @@ -201,7 +201,7 @@ def pre_test_win(){ ''' bat ''' cd %WIN_COMMUNITY_ROOT% - git clean -fxd + git clean -f git reset --hard git remote prune origin git fetch