From 41b9b98e3c4e11df8492335516be7d3083d76af1 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 6 Apr 2021 18:06:59 +0800 Subject: [PATCH 1/6] [TD-3652] add case for TD-3652 to resolve TD-3590 --- tests/pytest/fulltest.sh | 2 + tests/pytest/query/queryStddevWithGroupby.py | 68 ++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/pytest/query/queryStddevWithGroupby.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 7ff41b13a6..3266a1ab8f 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -22,6 +22,7 @@ python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f insert/before_1970.py python3 bug2265.py +python3 ./test.py -f insert/bug3654.py #table python3 ./test.py -f table/alter_wal0.py @@ -216,6 +217,7 @@ python3 ./test.py -f query/floatCompare.py python3 ./test.py -f query/query1970YearsAf.py python3 ./test.py -f query/bug3351.py python3 ./test.py -f query/bug3375.py +python3 ./test.py -f query/queryStddevWithGroupby.py diff --git a/tests/pytest/query/queryStddevWithGroupby.py b/tests/pytest/query/queryStddevWithGroupby.py new file mode 100644 index 0000000000..aee88ca9c5 --- /dev/null +++ b/tests/pytest/query/queryStddevWithGroupby.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def querysqls(self): + tdSql.query("select stddev(c1) from t10 group by c1") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 0) + tdSql.checkData(1, 0, 0) + tdSql.checkData(2, 0, 0) + tdSql.checkData(3, 0, 0) + tdSql.checkData(4, 0, 0) + tdSql.checkData(5, 0, 0) + tdSql.query("select stddev(c2) from t10") + tdSql.checkData(0, 0, 0.5) + + def run(self): + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + + tdLog.printNoPrefix("==========step1:create table && insert data") + tdSql.execute("create stable stb1 (ts timestamp , c1 int ,c2 float) tags(t1 int)") + tdSql.execute("create table t10 using stb1 tags(1)") + tdSql.execute("insert into t10 values ('1969-12-31 00:00:00.000', 2,1)") + tdSql.execute("insert into t10 values ('1970-01-01 00:00:00.000', 3,1)") + tdSql.execute("insert into t10 values (0, 4,1)") + tdSql.execute("insert into t10 values (now-18725d, 1,2)") + tdSql.execute("insert into t10 values ('2021-04-06 00:00:00.000', 5,2)") + tdSql.execute("insert into t10 values (now+1d,6,2)") + + tdLog.printNoPrefix("==========step2:query and check") + self.querysqls() + + tdLog.printNoPrefix("==========step3:after wal,check again") + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + self.querysqls() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 9f858447f74317054e0e67e19b4fccc8ec413702 Mon Sep 17 00:00:00 2001 From: wu champion Date: Tue, 6 Apr 2021 18:09:13 +0800 Subject: [PATCH 2/6] Update fulltest.sh --- tests/pytest/fulltest.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 3266a1ab8f..9d52b76a67 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -22,7 +22,6 @@ python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f insert/before_1970.py python3 bug2265.py -python3 ./test.py -f insert/bug3654.py #table python3 ./test.py -f table/alter_wal0.py From 891cf50ce9f425791aafb2c80472c1f670ae722d Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 7 Apr 2021 12:51:52 +0800 Subject: [PATCH 3/6] [TD-3579]: sort mnode syncCfg nodeInfo to fix leader election failure --- src/mnode/src/mnodeSdb.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 381cb11952..b470dd6359 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -315,6 +315,10 @@ void sdbUpdateAsync() { taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsSdbTmr); } +static int node_cmp(const void *l, const void *r) { + return ((SNodeInfo *)l)->nodeId - ((SNodeInfo *)r)->nodeId; +} + int32_t sdbUpdateSync(void *pMnodes) { SMInfos *pMinfos = pMnodes; if (!mnodeIsRunning()) { @@ -382,6 +386,8 @@ int32_t sdbUpdateSync(void *pMnodes) { return TSDB_CODE_SUCCESS; } + qsort(syncCfg.nodeInfo, syncCfg.replica, sizeof(syncCfg.nodeInfo[0]), node_cmp); + sdbInfo("vgId:1, work as mnode, replica:%d", syncCfg.replica); for (int32_t i = 0; i < syncCfg.replica; ++i) { sdbInfo("vgId:1, mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, @@ -1131,4 +1137,4 @@ static void *sdbWorkerFp(void *pWorker) { int32_t sdbGetReplicaNum() { return tsSdbMgmt.cfg.replica; -} \ No newline at end of file +} From b0d979ff326ac0e7ab5ad36e11f054c1be5b4e5f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Apr 2021 14:02:31 +0800 Subject: [PATCH 4/6] [TD-3607]: fix taosdemo limit and offset. (#5707) if offset+limit > count. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index edd7b86b1b..713b39d98b 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3527,18 +3527,18 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); - if (childTbl_limit) { + if ((childTbl_limit) && (g_Dbs.db[i].drop != true)) { if (childTbl_limit->type != cJSON_Number) { printf("ERROR: failed to read json, childtable_limit\n"); goto PARSE_OVER; } g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; } else { - g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result + g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid. } cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); - if (childTbl_offset) { + if ((childTbl_offset) && (g_Dbs.db[i].drop != true)) { if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) { printf("ERROR: failed to read json, childtable_offset\n"); goto PARSE_OVER; @@ -5170,7 +5170,9 @@ static void startMultiThreadInsertData(int threads, char* db_name, if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS) && (superTblInfo->childTblOffset >= 0)) { - if (superTblInfo->childTblLimit < 0) { + if ((superTblInfo->childTblLimit < 0) + || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) + > (superTblInfo->childTblCount))) { superTblInfo->childTblLimit = superTblInfo->childTblCount - superTblInfo->childTblOffset; } From 568a3ebc87a99e07e0e50633d97c44256861c037 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 7 Apr 2021 17:55:54 +0800 Subject: [PATCH 5/6] Hotfix/sangshuduo/td 3607 for master (#5712) * test * [TD-3677]: test pr message 1 * [TD-3671]change target branch * [TD-3677]: test pr message 2 * [TD-3677]: test pr message 3 * Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5688) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. Co-authored-by: Shuduo Sang * remove useless file * fix changing target branch * fix * fix * [TD-3607]: fix taosdemo limit and offset. if offset+limit > count. * [TD-3607]: taosdemo limit and offset. if child tbl not exist, dont take limit and offset value. Co-authored-by: huili <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: liuyq-617 Co-authored-by: plum-lihui Co-authored-by: Elias Soong Co-authored-by: Shuduo Sang --- Jenkinsfile | 2 +- src/kit/taosdemo/taosdemo.c | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d6939af31..dfe9ed4389 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -82,9 +82,9 @@ def pre_test(){ } } sh ''' - cd ${WK} git pull >/dev/null + export TZ=Asia/Harbin date git clean -dfx diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 713b39d98b..30f096954c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3479,9 +3479,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { - if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { + if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3)) + && (g_Dbs.db[i].drop == false)) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; - } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { + } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2) + || (g_Dbs.db[i].drop == true))) { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; } else { g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; @@ -3527,7 +3529,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); - if ((childTbl_limit) && (g_Dbs.db[i].drop != true)) { + if ((childTbl_limit) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { if (childTbl_limit->type != cJSON_Number) { printf("ERROR: failed to read json, childtable_limit\n"); goto PARSE_OVER; @@ -3538,7 +3541,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); - if ((childTbl_offset) && (g_Dbs.db[i].drop != true)) { + if ((childTbl_offset) && (g_Dbs.db[i].drop != true) + && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) { if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) { printf("ERROR: failed to read json, childtable_offset\n"); goto PARSE_OVER; From bbf49cb8c63932825e727ec847eb62bfdb3dffdc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Apr 2021 10:50:26 +0800 Subject: [PATCH 6/6] [TD-3683]: reduce buffer size for more stable table creation. (#5720) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 30f096954c..8c0c755113 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2573,10 +2573,7 @@ static void* createTable(void *sarg) int64_t lastPrintTime = taosGetTimestampMs(); int buff_len; - if (superTblInfo) - buff_len = superTblInfo->maxSqlLen; - else - buff_len = BUFFER_SIZE; + buff_len = BUFFER_SIZE / 8; char *buffer = calloc(buff_len, 1); if (buffer == NULL) { @@ -2624,7 +2621,7 @@ static void* createTable(void *sarg) return NULL; } len += snprintf(buffer + len, - superTblInfo->maxSqlLen - len, + buff_len - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, @@ -2632,7 +2629,7 @@ static void* createTable(void *sarg) free(tagsValBuf); batchNum++; if ((batchNum < superTblInfo->batchCreateTableNum) - && ((superTblInfo->maxSqlLen - len) + && ((buff_len - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) { continue; } @@ -5174,8 +5171,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS) && (superTblInfo->childTblOffset >= 0)) { - if ((superTblInfo->childTblLimit < 0) - || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) + if ((superTblInfo->childTblLimit < 0) + || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit) > (superTblInfo->childTblCount))) { superTblInfo->childTblLimit = superTblInfo->childTblCount - superTblInfo->childTblOffset;