From 88973b54606c2fa14ece5ccacd2a96fbaee3da62 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Tue, 6 Apr 2021 18:26:55 +0800 Subject: [PATCH 01/32] TD-3675 --- src/query/src/qAggMain.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index f18d093b89..e47545da95 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2771,14 +2771,16 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + pInfo->stage += 1; + // all data are null, set it completed if (pInfo->numOfElems == 0) { pResInfo->complete = true; + + return; } else { pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); } - - pInfo->stage += 1; } // the first stage, only acquire the min/max value @@ -2857,14 +2859,16 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + pInfo->stage += 1; + // all data are null, set it completed if (pInfo->numOfElems == 0) { pResInfo->complete = true; + + return; } else { pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); } - - pInfo->stage += 1; } if (pInfo->stage == 0) { From ba3fbea58c29778c78bb8c56a2cb4ec178e24ad0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 7 Apr 2021 13:19:19 +0800 Subject: [PATCH 02/32] [TD-3676]: add test case --- tests/pytest/fulltest.sh | 2 + tests/pytest/topic/topicQuery.py | 82 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/pytest/topic/topicQuery.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index cc6eb719f2..1746f8d2db 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -258,6 +258,8 @@ python3 test.py -f subscribe/singlemeter.py #python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py +# topic +python3 ./test.py -f topic/topicQuery.py #======================p3-end=============== #======================p4-start=============== diff --git a/tests/pytest/topic/topicQuery.py b/tests/pytest/topic/topicQuery.py new file mode 100644 index 0000000000..d4f8c4127d --- /dev/null +++ b/tests/pytest/topic/topicQuery.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def run(self): + tdSql.prepare() + + # test case for https://jira.taosdata.com:18080/browse/TD-3679 + print("==============step1") + tdSql.execute( + "create topic tq_test partitions 10") + tdSql.execute( + "insert into tq_test.p1(off, ts, content) values(0, %d, 'aaaa')" % self.ts) + tdSql.execute( + "insert into tq_test.p1(off, ts, content) values(1, %d, 'aaaa')" % (self.ts + 1)) + tdSql.execute( + "insert into tq_test.p1(off, ts, content) values(2, %d, 'aaaa')" % (self.ts + 2)) + tdSql.execute( + "insert into tq_test.p1(off, ts, content) values(3, %d, 'aaaa')" % (self.ts + 3)) + + print("==============step2") + + tdSql.query("select * from tq_test.p1") + tdSql.checkRows(4) + + tdSql.query("select * from tq_test.p1 where ts >= %d" % self.ts) + tdSql.checkRows(4) + + tdSql.query("select * from tq_test.p1 where ts > %d" % self.ts) + tdSql.checkRows(3) + + tdSql.query("select * from tq_test.p1 where ts = %d" % self.ts) + tdSql.checkRows(1) + + + tdSql.execute("use db") + tdSql.execute("create table test(ts timestamp, value int)") + tdSql.execute("insert into test values(%d, 1)" % self.ts) + tdSql.execute("insert into test values(%d, 1)" % (self.ts + 1)) + tdSql.execute("insert into test values(%d, 1)" % (self.ts + 2)) + tdSql.execute("insert into test values(%d, 1)" % (self.ts + 3)) + + tdSql.query("select * from test") + tdSql.checkRows(4) + + tdSql.query("select * from test where ts >= %d" % self.ts) + tdSql.checkRows(4) + + tdSql.query("select * from test where ts > %d" % self.ts) + tdSql.checkRows(3) + + tdSql.query("select * from test where ts = %d" % self.ts) + tdSql.checkRows(1) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From fdb5c5adc15d0f79f76489e83c1447249e76af44 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 7 Apr 2021 19:23:47 +0800 Subject: [PATCH 03/32] fix mem leak issue --- src/query/src/qExecutor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bd5fdda0f9..dda67d77b2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4015,7 +4015,7 @@ static SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, in return pFillCol; } -int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *tsdb, int32_t vgId, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4026,8 +4026,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *ts pQuery->timeWindowInterpo = timeWindowInterpoRequired(pQuery); pQuery->stabledev = isStabledev(pQuery); - pRuntimeEnv->prevResult = prevResult; - setScanLimitationByResultBuffer(pQuery); int32_t code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); @@ -6383,6 +6381,8 @@ int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *p SArray* prevResult = NULL; if (pQueryMsg->prevResultLen > 0) { prevResult = interResFromBinary(param->prevResult, pQueryMsg->prevResultLen); + + pRuntimeEnv->prevResult = prevResult; } pQuery->precision = tsdbGetCfg(tsdb)->precision; @@ -6404,7 +6404,7 @@ int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *p } // filter the qualified - if ((code = doInitQInfo(pQInfo, pTsBuf, prevResult, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTsBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } From 0eac169fb2ed6b0084e05133e8a468528164d950 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Wed, 7 Apr 2021 19:35:01 +0800 Subject: [PATCH 04/32] fix crash issue --- src/kit/shell/inc/shell.h | 2 +- src/kit/shell/src/shellCommand.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index d0b7149541..2374150c52 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -27,7 +27,7 @@ #define MAX_IP_SIZE 20 #define MAX_PASSWORD_SIZE 20 #define MAX_HISTORY_SIZE 1000 -#define MAX_COMMAND_SIZE 65536 +#define MAX_COMMAND_SIZE 1048586 #define HISTORY_FILE ".taos_history" #define DEFAULT_RES_SHOW_NUM 100 diff --git a/src/kit/shell/src/shellCommand.c b/src/kit/shell/src/shellCommand.c index 16545a5fe8..9173ab0efd 100644 --- a/src/kit/shell/src/shellCommand.c +++ b/src/kit/shell/src/shellCommand.c @@ -238,7 +238,7 @@ void resetCommand(Command *cmd, const char s[]) { clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); memset(cmd->buffer, 0, MAX_COMMAND_SIZE); memset(cmd->command, 0, MAX_COMMAND_SIZE); - strcpy(cmd->command, s); + strncpy(cmd->command, s, MAX_COMMAND_SIZE); int size = 0; int width = 0; getMbSizeInfo(s, &size, &width); From af90320a898372ea607f0fa8f45f991b4832ec8e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Apr 2021 13:07:18 +0800 Subject: [PATCH 05/32] Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5723) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] : taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] : taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] : taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] : fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] : taosdemo buffer overflow. adjust sample data test case. * test * [TD-3677]: test pr message 1 * [TD-3671]change target branch * [TD-3677]: test pr message 2 * [TD-3677]: test pr message 3 * Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5688) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. Co-authored-by: Shuduo Sang * remove useless file * fix changing target branch * fix * fix * [TD-3607]: taosdemo limit and offset. if limit+offset > count * Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5706) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] : taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] : taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] : taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] : fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] : taosdemo buffer overflow. adjust sample data test case. * [TD-3607]: taosdemo limit and offset. if limit+offset > count Co-authored-by: Shuduo Sang * [TD-3607]: taosdemo limit and offset. if child tbl not exist, dont take limit and offset value. * Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5713) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] : taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] : taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] : taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] : fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] : taosdemo buffer overflow. adjust sample data test case. * [TD-3607]: taosdemo limit and offset. if limit+offset > count * [TD-3607]: taosdemo limit and offset. if child tbl not exist, dont take limit and offset value. Co-authored-by: Shuduo Sang * fix taosdemo limit invalid warning condition. * [TD-3683]: reduce buffer size for more stable table creation. (#5719) Co-authored-by: Shuduo Sang Co-authored-by: Shuduo Sang Co-authored-by: huili <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: liuyq-617 Co-authored-by: plum-lihui Co-authored-by: Elias Soong Co-authored-by: Shengliang Guan --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 8c0c755113..59b31d6172 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5165,7 +5165,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, int limit, offset; if ((superTblInfo->childTblExists == TBL_NO_EXISTS) && - ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) { + ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) { printf("WARNING: offset and limit will not be used since the child tables are not exists!\n"); } From f81b77209b59f5aed329c01bc10d956019ac6f5e Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 8 Apr 2021 15:26:20 +0800 Subject: [PATCH 06/32] TD-3707 --- src/client/src/tscSQLParser.c | 2 +- tests/script/general/parser/groupby.sim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5841aa0cd5..e5a55cdfd3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6166,7 +6166,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } // projection query on super table does not compatible with "group by" syntax - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (tscIsProjectionQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index dd7331054c..124e76e85c 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -220,6 +220,7 @@ sql_error select sum(c3), ts, c2 from group_tb0 where c1 < 20 group by c1; sql_error select sum(c3), first(ts), c2 from group_tb0 where c1 < 20 group by c1; sql_error select first(c3), ts, c1, c2 from group_tb0 where c1 < 20 group by c1; sql_error select first(c3), last(c3), ts, c1 from group_tb0 where c1 < 20 group by c1; +sql_error select ts from group_tb0 group by c1; #===========================interval=====not support====================== sql_error select count(*), c1 from group_tb0 where c1<20 interval(1y) group by c1; From ad8604202837e8b72978e48f849cf8171b20f27b Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Thu, 8 Apr 2021 18:44:06 +0800 Subject: [PATCH 07/32] fix bug --- src/query/src/qFilterfunc.c | 4 ++-- src/tsdb/src/tsdbRead.c | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index 884f7e653f..dabce88423 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -124,7 +124,7 @@ bool greaterEqualOperator(SColumnFilterElem *pFilter, const char *minval, const bool equalOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { + if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) { int64_t minv = -1, maxv = -1; GET_TYPED_DATA(minv, int64_t, type, minval); GET_TYPED_DATA(maxv, int64_t, type, maxval); @@ -202,7 +202,7 @@ bool likeOperator(SColumnFilterElem *pFilter, const char *minval, const char *ma bool notEqualOperator(SColumnFilterElem *pFilter, const char *minval, const char *maxval, int16_t type) { SColumnFilterInfo *pFilterInfo = &pFilter->filterInfo; - if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { + if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_TIMESTAMP) { int64_t minv = -1, maxv = -1; GET_TYPED_DATA(minv, int64_t, type, minval); GET_TYPED_DATA(maxv, int64_t, type, maxval); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ea72760568..cd97b2a9d6 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2861,12 +2861,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta if (pHandle->statis[i].numOfNull == -1) { // set the column data are all NULL pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } - - SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i); - if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) { - pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst; - pHandle->statis[i].max = pBlockInfo->compBlock->keyLast; - } } int64_t elapsed = taosGetTimestampUs() - stime; From 0e72f1cb6d64ab34a8d712a5928b3eee65af1e71 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 8 Apr 2021 19:01:28 +0800 Subject: [PATCH 08/32] [###################################################################] --- tests/pytest/insert/basic.py | 3 +++ tests/pytest/topic/topicQuery.py | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/pytest/insert/basic.py b/tests/pytest/insert/basic.py index dcb5834d55..f23f38651a 100644 --- a/tests/pytest/insert/basic.py +++ b/tests/pytest/insert/basic.py @@ -43,6 +43,9 @@ class TDTestCase: tdSql.query("select * from tb") tdSql.checkRows(insertRows + 4) + # test case for https://jira.taosdata.com:18080/browse/TD-3716: + tdSql.error("insert into tb(now, 1)") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/topic/topicQuery.py b/tests/pytest/topic/topicQuery.py index d4f8c4127d..1ee3c3a427 100644 --- a/tests/pytest/topic/topicQuery.py +++ b/tests/pytest/topic/topicQuery.py @@ -54,11 +54,11 @@ class TDTestCase: tdSql.execute("use db") - tdSql.execute("create table test(ts timestamp, value int)") - tdSql.execute("insert into test values(%d, 1)" % self.ts) - tdSql.execute("insert into test values(%d, 1)" % (self.ts + 1)) - tdSql.execute("insert into test values(%d, 1)" % (self.ts + 2)) - tdSql.execute("insert into test values(%d, 1)" % (self.ts + 3)) + tdSql.execute("create table test(ts timestamp, start timestamp, value int)") + tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts, self.ts)) + tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 1, self.ts + 1)) + tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 2, self.ts + 2)) + tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 3, self.ts + 3)) tdSql.query("select * from test") tdSql.checkRows(4) @@ -72,6 +72,15 @@ class TDTestCase: tdSql.query("select * from test where ts = %d" % self.ts) tdSql.checkRows(1) + tdSql.query("select * from test where start >= %d" % self.ts) + tdSql.checkRows(4) + + tdSql.query("select * from test where start > %d" % self.ts) + tdSql.checkRows(3) + + tdSql.query("select * from test where start = %d" % self.ts) + tdSql.checkRows(1) + def stop(self): tdSql.close() @@ -79,4 +88,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 590b901518a1bdbf32ad42b6bae13a1a3462b9a7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Apr 2021 21:52:07 +0800 Subject: [PATCH 09/32] Feature/sangshuduo/td 3408 taosdemo async query (#5731) * test * [TD-3677]: test pr message 1 * [TD-3671]change target branch * [TD-3677]: test pr message 2 * [TD-3677]: test pr message 3 * Hotfix/sangshuduo/td 3197 fix taosdemo coverity scan (#5688) * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix taosdemo coverity scan issue. fix subscribeTest pids uninitialized. * [TD-3197] : fix taosdemo coverity scan issues. * [TD-3197] : fix coverity scan issues. check super tbl info pointer. * [TD-3197] : fix coverity scan issues. move sub tbl query thread join into loop * [TD-3197] : fix coverity scan issues. remove unused variable * [TD-3197] : fix coverity scan issues. use more secure random library * [TD-3197] : fix coverity scan issues. use strncpy for more safe * [TD-3197] : fix taosdemo coverity scan issue. replace arc4random with rand(). * [TD-3197] : fix coverity scan issues. check stb info pointer for start time * [TD-3197] : fix coverity scan issues. fix strcpy vulnerability * [TD-3197] : fix taosdemo coverity scan issue. modify taosdemoTest2. try to check database continously. * [TD-3197] : taosdemo coverity scan issues. * [TD-3197] : fix memory leak when parsing arguments. * [TD-3197] : fix cmake strip arguments. * [TD-3197] : taosdemo coverity scan. fix cmake string manipulation. Co-authored-by: Shuduo Sang * remove useless file * fix changing target branch * fix * fix * Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5706) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] : taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] : taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] : taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] : fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] : taosdemo buffer overflow. adjust sample data test case. * [TD-3607]: taosdemo limit and offset. if limit+offset > count Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5713) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] : taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] : taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] : taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] : fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] : taosdemo buffer overflow. adjust sample data test case. * [TD-3607]: taosdemo limit and offset. if limit+offset > count * [TD-3607]: taosdemo limit and offset. if child tbl not exist, dont take limit and offset value. Co-authored-by: Shuduo Sang * [TD-3683]: reduce buffer size for more stable table creation. (#5719) Co-authored-by: Shuduo Sang * [TD-3408]: taosdemo support async query. * [TD-3408]: taosdemo support async query. refactor * [TD-3408]: taosdemo support async query. refactor 2 * [TD-3408]: taosdemo support async query. refactor 3 * [TD-3408]: taosdemo support async query. refactor 4 * [TD-3408]: taosdemo support specified sql more than one line. Co-authored-by: huili <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: liuyq-617 Co-authored-by: plum-lihui Co-authored-by: Elias Soong Co-authored-by: Shuduo Sang Co-authored-by: Shengliang Guan --- src/kit/taosdemo/taosdemo.c | 502 +++++++++++++++++++++--------------- 1 file changed, 287 insertions(+), 215 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 59b31d6172..6939f5dae5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -67,6 +67,12 @@ enum TEST_MODE { INVAID_TEST }; +enum QUERY_MODE { + SYNC_QUERY_MODE, // 0 + ASYNC_QUERY_MODE, // 1 + INVALID_MODE +}; + #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define MAX_USERNAME_SIZE 64 @@ -198,7 +204,7 @@ typedef struct SArguments_S { bool verbose_print; bool performance_print; char * output_file; - int mode; + int query_mode; char * datatype[MAX_NUM_DATATYPE + 1]; int len_of_binary; int num_of_CPR; @@ -351,7 +357,7 @@ typedef struct SpecifiedQueryInfo_S { int rate; // 0: unlimit > 0 loop/s int concurrent; int sqlCount; - int subscribeMode; // 0: sync, 1: async + int mode; // 0: sync, 1: async int subscribeInterval; // ms int queryTimes; int subscribeRestart; @@ -365,7 +371,7 @@ typedef struct SuperQueryInfo_S { char sTblName[MAX_TB_NAME_SIZE+1]; int rate; // 0: unlimit > 0 loop/s int threadCnt; - int subscribeMode; // 0: sync, 1: async + int mode; // 0: sync, 1: async int subscribeInterval; // ms int subscribeRestart; int subscribeKeepProgress; @@ -429,6 +435,8 @@ typedef struct SThreadInfo_S { int64_t maxDelay; int64_t minDelay; + // query + int querySeq; // sequence number of sql command } threadInfo; #ifdef WINDOWS @@ -714,7 +722,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else if (strcmp(argv[i], "-s") == 0) { arguments->sqlFile = argv[++i]; } else if (strcmp(argv[i], "-q") == 0) { - arguments->mode = atoi(argv[++i]); + arguments->query_mode = atoi(argv[++i]); } else if (strcmp(argv[i], "-T") == 0) { arguments->num_of_threads = atoi(argv[++i]); } else if (strcmp(argv[i], "-i") == 0) { @@ -758,7 +766,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { char *dupstr = strdup(argv[i]); char *running = dupstr; char *token = strsep(&running, ","); - while (token != NULL) { + while(token != NULL) { if (strcasecmp(token, "INT") && strcasecmp(token, "FLOAT") && strcasecmp(token, "TINYINT") @@ -964,7 +972,7 @@ static void getResult(TAOS_RES *res, char* resultFileName) { char temp[16000]; // fetch the records row by row - while ((row = taos_fetch_row(res))) { + while((row = taos_fetch_row(res))) { if (totalLen >= 100*1024*1024 - 32000) { if (fp) fprintf(fp, "%s", databuf); totalLen = 0; @@ -986,7 +994,8 @@ static void getResult(TAOS_RES *res, char* resultFileName) { static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { TAOS_RES *res = taos_query(taos, command); if (res == NULL || taos_errno(res) != 0) { - printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res)); + errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", + __func__, __LINE__, command, taos_errstr(res)); taos_free_result(res); return; } @@ -1163,7 +1172,8 @@ static int printfInsertMeta() { if (g_Dbs.db[i].dbCfg.precision[0] != 0) { if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + printf(" precision: \033[33m%s\033[0m\n", + g_Dbs.db[i].dbCfg.precision); } else { printf("\033[1m\033[40;31m precision error: %s\033[0m\n", g_Dbs.db[i].dbCfg.precision); @@ -1171,11 +1181,13 @@ static int printfInsertMeta() { } } - printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); + printf(" super table count: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTblCount); for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { printf(" super table[\033[33m%d\033[0m]:\n", j); - printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); + printf(" stbName: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sTblName); if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); @@ -1241,7 +1253,7 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].sampleFile); printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); - printf(" columnCount: \033[33m%d\033[0m\n ", + printf(" columnCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].columnCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); @@ -1459,41 +1471,61 @@ static void printfQueryMeta() { printf("\n"); printf("specified table query info: \n"); - printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.rate); + printf("query interval: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.rate); printf("top query times:\033[33m%d\033[0m\n", g_args.query_times); - printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); - printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount); + printf("concurrent: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.concurrent); + printf("sqlCount: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.sqlCount); printf("specified tbl query times:\n"); - printf(" \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.queryTimes); + printf(" \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.queryTimes); if (SUBSCRIBE_TEST == g_args.test_mode) { - printf("mod: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeMode); - printf("interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval); - printf("restart: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeRestart); - printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + printf("mod: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.mode); + printf("interval: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); } for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.specifiedQueryInfo.sql[i]); + printf(" sql[%d]: \033[33m%s\033[0m\n", + i, g_queryInfo.specifiedQueryInfo.sql[i]); } printf("\n"); - printf("super table query info: \n"); - printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); - printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); - printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); - printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); - printf("stb query times:\033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.queryTimes); + printf("super table query info:\n"); + printf("query interval: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.rate); + printf("threadCnt: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.threadCnt); + printf("childTblCount: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.childTblCount); + printf("stable name: \033[33m%s\033[0m\n", + g_queryInfo.superQueryInfo.sTblName); + printf("stb query times:\033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.queryTimes); if (SUBSCRIBE_TEST == g_args.test_mode) { - printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); - printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); - printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); - printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); + printf("mod: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.mode); + printf("interval: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.subscribeKeepProgress); } - printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); + printf("sqlCount: \033[33m%d\033[0m\n", + g_queryInfo.superQueryInfo.sqlCount); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); + printf(" sql[%d]: \033[33m%s\033[0m\n", + i, g_queryInfo.superQueryInfo.sql[i]); } printf("\n"); @@ -1637,7 +1669,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { + while((row = taos_fetch_row(res)) != NULL) { // sys database name : 'log' if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) { @@ -1670,7 +1702,8 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); - dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); + dbInfos[count]->cachelast = + (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); tstrncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], @@ -1681,7 +1714,8 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { count++; if (count > MAX_DATABASE_COUNT) { - errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT); + errorPrint("%s() LN%d, The database count overflow than %d\n", + __func__, __LINE__, MAX_DATABASE_COUNT); break; } } @@ -1691,6 +1725,7 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { static void printfDbInfoForQueryToFile( char* filename, SDbInfo* dbInfos, int index) { + if (filename[0] == 0) return; @@ -1909,7 +1944,7 @@ static int postProceSql(char* host, uint16_t port, char* sqlstr) if (bytes == 0) break; sent+=bytes; - } while (sent < req_str_len); + } while(sent < req_str_len); memset(response_buf, 0, RESP_BUF_LEN); resp_len = sizeof(response_buf) - 1; @@ -1927,7 +1962,7 @@ static int postProceSql(char* host, uint16_t port, char* sqlstr) if (bytes == 0) break; received += bytes; - } while (received < resp_len); + } while(received < resp_len); if (received == resp_len) { free(request_buf); @@ -1951,7 +1986,8 @@ static int postProceSql(char* host, uint16_t port, char* sqlstr) static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { - errorPrint("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1); + errorPrint("%s() LN%d, calloc failed! size:%d\n", + __func__, __LINE__, TSDB_MAX_SQL_LEN+1); return NULL; } @@ -2155,7 +2191,7 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, } char* pTblName = childTblName; - while ((row = taos_fetch_row(res)) != NULL) { + while((row = taos_fetch_row(res)) != NULL) { int32_t* len = taos_fetch_lengths(res); tstrncpy(pTblName, (char *)row[0], len[0]+1); //printf("==== sub table name: %s\n", pTblName); @@ -2218,7 +2254,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, int tagIndex = 0; int columnIndex = 0; TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { + while((row = taos_fetch_row(res)) != NULL) { if (0 == count) { count++; continue; @@ -2765,7 +2801,7 @@ static void createChildTables() { // normal table len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); int j = 0; - while (g_args.datatype[j]) { + while(g_args.datatype[j]) { if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { @@ -2824,7 +2860,7 @@ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { return -1; } - while ((readLen = tgetline(&line, &n, fp)) != -1) { + while((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; } @@ -2888,7 +2924,7 @@ static int readSampleFromCsvFileToMem( assert(superTblInfo->sampleDataBuf); memset(superTblInfo->sampleDataBuf, 0, MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow); - while (1) { + while(1) { readLen = tgetline(&line, &n, fp); if (-1 == readLen) { if(0 != fseek(fp, 0, SEEK_SET)) { @@ -2967,7 +3003,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( if (countObj && countObj->type == cJSON_Number) { count = countObj->valueint; } else if (countObj && countObj->type != cJSON_Number) { - errorPrint("%s() LN%d, failed to read json, column count not found\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, column count not found\n", + __func__, __LINE__); goto PARSE_OVER; } else { count = 1; @@ -2976,8 +3013,10 @@ static bool getColumnAndTagTypeFromInsertJsonFile( // column info memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(column, "type"); - if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - errorPrint("%s() LN%d: failed to read json, column type not found\n", __func__, __LINE__); + if (!dataType || dataType->type != cJSON_String + || dataType->valuestring == NULL) { + errorPrint("%s() LN%d: failed to read json, column type not found\n", + __func__, __LINE__); goto PARSE_OVER; } //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); @@ -2987,7 +3026,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( if (dataLen && dataLen->type == cJSON_Number) { columnCase.dataLen = dataLen->valueint; } else if (dataLen && dataLen->type != cJSON_Number) { - debugPrint("%s() LN%d: failed to read json, column len not found\n", __func__, __LINE__); + debugPrint("%s() LN%d: failed to read json, column len not found\n", + __func__, __LINE__); goto PARSE_OVER; } else { columnCase.dataLen = 8; @@ -3007,13 +3047,15 @@ static bool getColumnAndTagTypeFromInsertJsonFile( // tags cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); if (!tags || tags->type != cJSON_Array) { - debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, tags not found\n", + __func__, __LINE__); goto PARSE_OVER; } int tagSize = cJSON_GetArraySize(tags); if (tagSize > MAX_TAG_COUNT) { - debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT); + errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", + __func__, __LINE__, MAX_TAG_COUNT); goto PARSE_OVER; } @@ -3036,8 +3078,10 @@ static bool getColumnAndTagTypeFromInsertJsonFile( // column info memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(tag, "type"); - if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - printf("ERROR: failed to read json, tag type not found\n"); + if (!dataType || dataType->type != cJSON_String + || dataType->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, tag type not found\n", + __func__, __LINE__); goto PARSE_OVER; } tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); @@ -3046,14 +3090,16 @@ static bool getColumnAndTagTypeFromInsertJsonFile( if (dataLen && dataLen->type == cJSON_Number) { columnCase.dataLen = dataLen->valueint; } else if (dataLen && dataLen->type != cJSON_Number) { - printf("ERROR: failed to read json, column len not found\n"); + errorPrint("%s() LN%d, failed to read json, column len not found\n", + __func__, __LINE__); goto PARSE_OVER; } else { columnCase.dataLen = 0; } for (int n = 0; n < count; ++n) { - tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, + MAX_TB_NAME_SIZE); superTbls->tags[index].dataLen = columnCase.dataLen; index++; } @@ -3063,9 +3109,6 @@ static bool getColumnAndTagTypeFromInsertJsonFile( ret = true; PARSE_OVER: - //free(content); - //cJSON_Delete(root); - //fclose(fp); return ret; } @@ -3142,7 +3185,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!gInsertInterval) { g_args.insert_interval = 0; } else { - errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3163,7 +3207,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!interlaceRows) { g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req } else { - errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3173,7 +3218,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!maxSqlLen) { g_args.max_sql_len = TSDB_PAYLOAD_SIZE; } else { - errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3183,7 +3229,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!numRecPerReq) { g_args.num_of_RPR = 0xffff; } else { - errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3509,7 +3556,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!dataSource) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); } else { - errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, data_source not found\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3584,7 +3632,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); - if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { + if (sampleFile && sampleFile->type == cJSON_String + && sampleFile->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, sampleFile->valuestring, MAX_FILE_NAME_LEN); } else if (!sampleFile) { @@ -3727,9 +3776,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { ret = true; PARSE_OVER: - //free(content); - //cJSON_Delete(root); - //fclose(fp); return ret; } @@ -3795,7 +3841,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!gQueryTimes) { g_args.query_times = 1; } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3833,35 +3880,45 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.specifiedQueryInfo.rate = 0; } - cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); + cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, + "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint; } else if (!specifiedQueryTimes) { g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times; } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); if (concurrent && concurrent->type == cJSON_Number) { g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint; + if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { + errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", + __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, + g_queryInfo.specifiedQueryInfo.concurrent); + goto PARSE_OVER; + } } else if (!concurrent) { g_queryInfo.specifiedQueryInfo.concurrent = 1; } - cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode"); - if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { - if (0 == strcmp("sync", mode->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeMode = 0; - } else if (0 == strcmp("async", mode->valuestring)) { - g_queryInfo.specifiedQueryInfo.subscribeMode = 1; + cJSON* queryMode = cJSON_GetObjectItem(specifiedQuery, "mode"); + if (queryMode && queryMode->type == cJSON_String + && queryMode->valuestring != NULL) { + if (0 == strcmp("sync", queryMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.mode = SYNC_QUERY_MODE; + } else if (0 == strcmp("async", queryMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.mode = ASYNC_QUERY_MODE; } else { - printf("ERROR: failed to read json, subscribe mod error\n"); + errorPrint("%s() LN%d, failed to read json, query mode input error\n", + __func__, __LINE__); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.subscribeMode = 0; + g_queryInfo.specifiedQueryInfo.mode = SYNC_QUERY_MODE; } cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); @@ -3908,12 +3965,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (!superSqls) { g_queryInfo.specifiedQueryInfo.sqlCount = 0; } else if (superSqls->type != cJSON_Array) { - printf("ERROR: failed to read json, super sqls not found\n"); + errorPrint("%s() LN%d, failed to read json, super sqls not found\n", + __func__, __LINE__); goto PARSE_OVER; } else { int superSqlSize = cJSON_GetArraySize(superSqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { - printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", + __func__, __LINE__, MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } @@ -3965,7 +4024,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } else if (!superQueryTimes) { g_queryInfo.superQueryInfo.queryTimes = g_args.query_times; } else { - errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -3984,25 +4044,30 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { //} cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname"); - if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { - tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); + if (stblname && stblname->type == cJSON_String + && stblname->valuestring != NULL) { + tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, + MAX_TB_NAME_SIZE); } else { - printf("ERROR: failed to read json, super table name not found\n"); + errorPrint("%s() LN%d, failed to read json, super table name input error\n", + __func__, __LINE__); goto PARSE_OVER; } cJSON* submode = cJSON_GetObjectItem(superQuery, "mode"); - if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { + if (submode && submode->type == cJSON_String + && submode->valuestring != NULL) { if (0 == strcmp("sync", submode->valuestring)) { - g_queryInfo.superQueryInfo.subscribeMode = 0; + g_queryInfo.superQueryInfo.mode = SYNC_QUERY_MODE; } else if (0 == strcmp("async", submode->valuestring)) { - g_queryInfo.superQueryInfo.subscribeMode = 1; + g_queryInfo.superQueryInfo.mode = ASYNC_QUERY_MODE; } else { - printf("ERROR: failed to read json, subscribe mod error\n"); + errorPrint("%s() LN%d, failed to read json, query mode input error\n", + __func__, __LINE__); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.subscribeMode = 0; + g_queryInfo.superQueryInfo.mode = SYNC_QUERY_MODE; } cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval"); @@ -4015,7 +4080,8 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart"); - if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { + if (subrestart && subrestart->type == cJSON_String + && subrestart->valuestring != NULL) { if (0 == strcmp("yes", subrestart->valuestring)) { g_queryInfo.superQueryInfo.subscribeRestart = 1; } else if (0 == strcmp("no", subrestart->valuestring)) { @@ -4049,12 +4115,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (!subsqls) { g_queryInfo.superQueryInfo.sqlCount = 0; } else if (subsqls->type != cJSON_Array) { - printf("ERROR: failed to read json, super sqls not found\n"); + errorPrint("%s() LN%d: failed to read json, super sqls not found\n", + __func__, __LINE__); goto PARSE_OVER; } else { int superSqlSize = cJSON_GetArraySize(subsqls); if (superSqlSize > MAX_QUERY_SQL_COUNT) { - printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n", + __func__, __LINE__, MAX_QUERY_SQL_COUNT); goto PARSE_OVER; } @@ -4064,19 +4132,25 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { if (sql == NULL) continue; cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); - if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("ERROR: failed to read json, sql not found\n"); + if (!sqlStr || sqlStr->type != cJSON_String + || sqlStr->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, sql not found\n", + __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, + MAX_QUERY_SQL_LENGTH); cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ - tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + if (result != NULL && result->type == cJSON_String + && result->valuestring != NULL){ + tstrncpy(g_queryInfo.superQueryInfo.result[j], + result->valuestring, MAX_FILE_NAME_LEN); } else if (NULL == result) { memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); } else { - printf("ERROR: failed to read json, sub query result file not found\n"); + errorPrint("%s() LN%d, failed to read json, sub query result file not found\n", + __func__, __LINE__); goto PARSE_OVER; } } @@ -4086,9 +4160,6 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { ret = true; PARSE_OVER: - //free(content); - //cJSON_Delete(root); - //fclose(fp); return ret; } @@ -5415,7 +5486,7 @@ static void *readTable(void *sarg) { return NULL; } - while (taos_fetch_row(pSql) != NULL) { + while(taos_fetch_row(pSql) != NULL) { count++; } @@ -5491,7 +5562,7 @@ static void *readMetric(void *sarg) { return NULL; } int count = 0; - while (taos_fetch_row(pSql) != NULL) { + while(taos_fetch_row(pSql) != NULL) { count++; } t = getCurrentTimeUs() - t; @@ -5602,7 +5673,7 @@ static int insertTestProcess() { return 0; } -static void *superQueryProcess(void *sarg) { +static void *specifiedQueryProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; if (winfo->taos == NULL) { @@ -5643,32 +5714,35 @@ static void *superQueryProcess(void *sarg) { } st = taosGetTimestampUs(); - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - int64_t t1 = taosGetTimestampUs(); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); - } - selectAndGetResult(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], tmpFile); - int64_t t2 = taosGetTimestampUs(); - printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", - taosGetSelfPthreadId(), (t2 - t1)/1000000.0); - } else { - int64_t t1 = taosGetTimestampUs(); - int retCode = postProceSql(g_queryInfo.host, - g_queryInfo.port, g_queryInfo.specifiedQueryInfo.sql[i]); - int64_t t2 = taosGetTimestampUs(); - printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", - taosGetSelfPthreadId(), (t2 - t1)/1000000.0); - if (0 != retCode) { - printf("====restful return fail, threadID[%d]\n", winfo->threadID); - return NULL; - } + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + int64_t t1 = taosGetTimestampUs(); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.specifiedQueryInfo.result[winfo->querySeq][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[winfo->querySeq], + winfo->threadID); + } + selectAndGetResult(winfo->taos, + g_queryInfo.specifiedQueryInfo.sql[winfo->querySeq], tmpFile); + int64_t t2 = taosGetTimestampUs(); + printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + } else { + int64_t t1 = taosGetTimestampUs(); + int retCode = postProceSql(g_queryInfo.host, + g_queryInfo.port, + g_queryInfo.specifiedQueryInfo.sql[winfo->querySeq]); + int64_t t2 = taosGetTimestampUs(); + printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + + if (0 != retCode) { + printf("====restful return fail, threadID[%d]\n", winfo->threadID); + return NULL; } } + et = taosGetTimestampUs(); printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); @@ -5698,7 +5772,7 @@ static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { //printf("3: %s\n", outSql); } -static void *subQueryProcess(void *sarg) { +static void *superQueryProcess(void *sarg) { char sqlstr[1024]; threadInfo *winfo = (threadInfo *)sarg; @@ -5791,43 +5865,45 @@ static int queryTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; //==== create sub threads for query from specify table - if (g_queryInfo.specifiedQueryInfo.sqlCount > 0 - && g_queryInfo.specifiedQueryInfo.concurrent > 0) { + int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent; + int nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount; - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - if (NULL == pids) { - taos_close(taos); - ERROR_EXIT("memory allocation failed\n"); - } - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); - if (NULL == infos) { + if ((nSqlCount > 0) && (nConcurrent > 0)) { + + pids = malloc(nConcurrent * nSqlCount * sizeof(pthread_t)); + infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo)); + + if ((NULL == pids) || (NULL == infos)) { taos_close(taos); - free(pids); ERROR_EXIT("memory allocation failed for create threads\n"); } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; + for (int i = 0; i < nConcurrent; i++) { + for (int j = 0; j < nSqlCount; j++) { + threadInfo *t_info = infos + i * nSqlCount + j; + t_info->threadID = i * nSqlCount + j; + t_info->querySeq = j; - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); - if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(taos); free(infos); free(pids); errorPrint( "use database %s failed!\n\n", g_queryInfo.dbName); return -1; + } } + + t_info->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + i * nSqlCount + j, NULL, specifiedQueryProcess, + t_info); } - - t_info->taos = NULL;// TODO: workaround to use separate taos connection; - - pthread_create(pids + i, NULL, superQueryProcess, t_info); } } else { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -5841,18 +5917,12 @@ static int queryTestProcess() { if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - if (NULL == pidsOfSub) { - free(infos); - free(pids); - - ERROR_EXIT("memory allocation failed for create threads\n"); - } - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); - if (NULL == infosOfSub) { - free(pidsOfSub); + + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { free(infos); free(pids); + ERROR_EXIT("memory allocation failed for create threads\n"); } @@ -5880,7 +5950,7 @@ static int queryTestProcess() { t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1; startFrom = t_info->end_table_to + 1; t_info->taos = NULL; // TODO: workaround to use separate taos connection; - pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); + pthread_create(pidsOfSub + i, NULL, superQueryProcess, t_info); } g_queryInfo.superQueryInfo.threadCnt = threads; @@ -5888,8 +5958,12 @@ static int queryTestProcess() { g_queryInfo.superQueryInfo.threadCnt = 0; } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); + if ((nSqlCount > 0) && (nConcurrent > 0)) { + for (int i = 0; i < nConcurrent; i++) { + for (int j = 0; j < nSqlCount; j++) { + pthread_join(pids[i * nSqlCount + j], NULL); + } + } } tmfree((char*)pids); @@ -5920,7 +5994,7 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { TAOS_SUB* tsub = NULL; - if (g_queryInfo.specifiedQueryInfo.subscribeMode) { + if (g_queryInfo.specifiedQueryInfo.mode) { tsub = taos_subscribe(taos, g_queryInfo.specifiedQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, @@ -5996,13 +6070,13 @@ static void *subSubscribeProcess(void *sarg) { } //et = taosGetTimestampMs(); //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while (0); + } while(0); // start loop to consume result TAOS_RES* res = NULL; - while (1) { + while(1) { for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (1 == g_queryInfo.superQueryInfo.subscribeMode) { + if (1 == g_queryInfo.superQueryInfo.mode) { continue; } @@ -6073,7 +6147,8 @@ static void *superSubscribeProcess(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); } - tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); + tsub[i] = subscribeImpl(winfo->taos, + g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { taos_close(winfo->taos); return NULL; @@ -6081,13 +6156,13 @@ static void *superSubscribeProcess(void *sarg) { } //et = taosGetTimestampMs(); //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while (0); + } while(0); // start loop to consume result TAOS_RES* res = NULL; - while (1) { + while(1) { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - if (1 == g_queryInfo.specifiedQueryInfo.subscribeMode) { + if (SYNC_QUERY_MODE == g_queryInfo.specifiedQueryInfo.mode) { continue; } @@ -6105,7 +6180,8 @@ static void *superSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); @@ -6308,7 +6384,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.queryMode = g_args.mode; + g_Dbs.queryMode = g_args.query_mode; g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; @@ -6410,7 +6486,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) double t = getCurrentTimeUs(); - while ((read_len = tgetline(&line, &line_len, fp)) != -1) { + while((read_len = tgetline(&line, &line_len, fp)) != -1) { if (read_len >= MAX_SQL_SIZE) continue; line[--read_len] = '\0'; @@ -6473,52 +6549,50 @@ static void testMetaFile() { } static void queryResult() { - // select - if (false == g_Dbs.insert_only) { - // query data + // query data - pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_from = 0; + pthread_t read_id; + threadInfo *rInfo = malloc(sizeof(threadInfo)); + assert(rInfo); + rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + rInfo->start_table_from = 0; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; - if (g_args.use_metric) { - rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; - rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - tstrncpy(rInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); - } else { - rInfo->ntables = g_args.num_of_tables; - rInfo->end_table_to = g_args.num_of_tables -1; - tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); - } + //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + if (g_args.use_metric) { + rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; + rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; + rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + tstrncpy(rInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); + } else { + rInfo->ntables = g_args.num_of_tables; + rInfo->end_table_to = g_args.num_of_tables -1; + tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + } - rInfo->taos = taos_connect( - g_Dbs.host, - g_Dbs.user, - g_Dbs.password, - g_Dbs.db[0].dbName, - g_Dbs.port); - if (rInfo->taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", - taos_errstr(NULL)); - free(rInfo); - exit(-1); - } + rInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + if (rInfo->taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); + free(rInfo); + exit(-1); + } - tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); + tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); - if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); - } else { - pthread_create(&read_id, NULL, readMetric, rInfo); - } - pthread_join(read_id, NULL); - taos_close(rInfo->taos); - free(rInfo); - } + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, rInfo); + } else { + pthread_create(&read_id, NULL, readMetric, rInfo); + } + pthread_join(read_id, NULL); + taos_close(rInfo->taos); + free(rInfo); } static void testCmdLine() { @@ -6536,9 +6610,7 @@ static void testCmdLine() { g_args.test_mode = INSERT_TEST; insertTestProcess(); - if (g_Dbs.insert_only) - return; - else + if (false == g_Dbs.insert_only) queryResult(); } From b7579334deed7b4c629ec977a6b8b4c7e3f47513 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 9 Apr 2021 02:53:30 +0000 Subject: [PATCH 10/32] [TD-3682]: Insufficient disk space may cause oom --- src/common/src/tglobal.c | 2 +- src/sync/src/syncMain.c | 11 +++++++++-- src/vnode/inc/vnodeInt.h | 1 + src/vnode/src/vnodeWrite.c | 24 ++++++++++++++++++++---- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 5f4ce046ed..ea5bf7954d 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -212,7 +212,7 @@ float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; float tsUsedDataDirGB = 0; float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 1.0f; +float tsMinimalDataDirGB = 2.0f; int32_t tsTotalMemoryMB = 0; uint32_t tsVersion = 0; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 72442eee6c..29ced21291 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -997,17 +997,24 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { sTrace("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); + int32_t code = 0; if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); + code = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { - syncSaveIntoBuffer(pPeer, pHead); + code = syncSaveIntoBuffer(pPeer, pHead); } else { sError("%s, forward discarded since sstatus:%s, hver:%" PRIu64, pPeer->id, syncStatus[nodeSStatus], pHead->version); + code = -1; } } + + if (code != 0) { + sError("%s, failed to process fwd msg, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); + syncRestartConnection(pPeer); + } } static void syncProcessPeersStatusMsg(SPeersStatus *pPeersStatus, SSyncPeer *pPeer) { diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 4aa07196a7..d770a38e37 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -37,6 +37,7 @@ extern int32_t vDebugFlag; typedef struct { int32_t vgId; // global vnode group ID int32_t refCount; // reference count + int64_t queuedWMsgSize; int32_t queuedWMsg; int32_t queuedRMsg; int32_t flowctrlLevel; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index a0be52db7a..aab685e678 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -25,6 +25,7 @@ #include "vnodeStatus.h" #define MAX_QUEUED_MSG_NUM 100000 +#define MAX_QUEUED_MSG_SIZE 1024*1024*1024 //1GB extern void * tsDnodeTmr; static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *); @@ -269,6 +270,13 @@ static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) { } } + if (tsAvailDataDirGB <= tsMinimalDataDirGB) { + vError("vgId:%d, failed to write into vwqueue since no diskspace, avail:%fGB", pVnode->vgId, tsAvailDataDirGB); + taosFreeQitem(pWrite); + vnodeRelease(pVnode); + return TSDB_CODE_VND_NO_DISKSPACE; + } + if (!vnodeInReadyOrUpdatingStatus(pVnode)) { vError("vgId:%d, failed to write into vwqueue, vstatus is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], pVnode->refCount, pVnode); @@ -278,14 +286,17 @@ static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) { } int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); - if (queued > MAX_QUEUED_MSG_NUM) { + int64_t queuedSize = atomic_add_fetch_64(&pVnode->queuedWMsgSize, pWrite->pHead.len); + + if (queued > MAX_QUEUED_MSG_NUM || queuedSize > MAX_QUEUED_MSG_SIZE) { int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3; if (ms > 100) ms = 100; vDebug("vgId:%d, too many msg:%d in vwqueue, flow control %dms", pVnode->vgId, queued, ms); taosMsleep(ms); } - vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); + vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d size:%" PRId64, pVnode->vgId, pVnode->refCount, + pVnode->queuedWMsg, pVnode->queuedWMsgSize); taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); return TSDB_CODE_SUCCESS; @@ -308,7 +319,10 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { SVnodeObj *pVnode = vparam; int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); - vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, queued); + int64_t queuedSize = atomic_sub_fetch_64(&pVnode->queuedWMsgSize, pWrite->pHead.len); + + vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d size:%" PRId64, pVnode->vgId, pWrite, + pWrite->rpcMsg.ahandle, queued, queuedSize); taosFreeQitem(pWrite); vnodeRelease(pVnode); @@ -344,7 +358,9 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { SVnodeObj *pVnode = pWrite->pVnode; if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; - if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; + if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->queuedWMsgSize < MAX_QUEUED_MSG_SIZE && + pVnode->flowctrlLevel <= 0) + return 0; if (tsEnableFlowCtrl == 0) { int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2); From 0a95895c9d4b284e65f5db370804238d86b24829 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 9 Apr 2021 10:59:03 +0800 Subject: [PATCH 11/32] fix taos crash issue --- src/client/src/tscParseInsert.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 926ee44b70..920937928f 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -937,6 +937,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC return ret; } + if (sql == NULL) { + return TSDB_CODE_TSC_INVALID_SQL; + } + code = tscGetTableMetaEx(pSql, pTableMetaInfo, true); if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { return code; @@ -945,6 +949,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } else { sql = sToken.z; + if (sql == NULL) { + return TSDB_CODE_TSC_INVALID_SQL; + } + code = tscGetTableMetaEx(pSql, pTableMetaInfo, false); if (pCmd->curSql == NULL) { assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS); @@ -952,10 +960,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } *sqlstr = sql; - - if (*sqlstr == NULL) { - code = TSDB_CODE_TSC_INVALID_SQL; - } return code; } From 3da3b0de53251fa42ff6f4044d575a54260b7897 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 9 Apr 2021 11:05:33 +0800 Subject: [PATCH 12/32] asyncdemo crash issue --- tests/examples/c/asyncdemo.c | 44 +++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c index d711ce22c1..f2a96dd825 100644 --- a/tests/examples/c/asyncdemo.c +++ b/tests/examples/c/asyncdemo.c @@ -28,7 +28,8 @@ int points = 5; int numOfTables = 3; -int tablesProcessed = 0; +int tablesInsertProcessed = 0; +int tablesSelectProcessed = 0; int64_t st, et; typedef struct { @@ -134,6 +135,9 @@ int main(int argc, char *argv[]) gettimeofday(&systemTime, NULL); st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + tablesInsertProcessed = 0; + tablesSelectProcessed = 0; + for (i = 0; iname); - tablesProcessed++; - if (tablesProcessed >= numOfTables) { + tablesInsertProcessed++; + if (tablesInsertProcessed >= numOfTables) { gettimeofday(&systemTime, NULL); et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; printf("%lld mseconds to insert %d data points\n", (et - st) / 1000, points*numOfTables); @@ -251,15 +263,17 @@ void taos_retrieve_call_back(void *param, TAOS_RES *tres, int numOfRows) //taos_free_result(tres); printf("%d rows data retrieved from %s\n", pTable->rowsRetrieved, pTable->name); - tablesProcessed++; - if (tablesProcessed >= numOfTables) { + tablesSelectProcessed++; + if (tablesSelectProcessed >= numOfTables) { gettimeofday(&systemTime, NULL); et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; printf("%lld mseconds to query %d data rows\n", (et - st) / 1000, points * numOfTables); } + + taos_free_result(tres); } - taos_free_result(tres); + } void taos_select_call_back(void *param, TAOS_RES *tres, int code) @@ -276,6 +290,4 @@ void taos_select_call_back(void *param, TAOS_RES *tres, int code) taos_cleanup(); exit(1); } - - taos_free_result(tres); } From ad073f168cace2fb7274a2ecd2ed1dd2ce290735 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 9 Apr 2021 13:32:13 +0800 Subject: [PATCH 13/32] [TD-3636]: fix taosdemo outorder range algorithm. (#5736) patch for master. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 48 +++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 6939f5dae5..e804d93619 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4520,22 +4520,23 @@ static int generateDataTail(char *tableName, int32_t tableSeq, pSamplePos); } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - int rand_num = taosRandom() % 100; - if (0 != superTblInfo->disorderRatio + int rand_num = taosRandom() % 100; + int randTail; + if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { - int64_t d = startTime - + superTblInfo->timeStampStep * k - - taosRandom() % superTblInfo->disorderRange; - retLen = generateRowData( + randTail = (superTblInfo->timeStampStep * k + + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %d\n", randTail); + } else { + randTail = superTblInfo->timeStampStep * k; + } + + uint64_t d = startTime + + randTail; + retLen = generateRowData( data, d, superTblInfo); - } else { - retLen = generateRowData( - data, - startTime + superTblInfo->timeStampStep * k, - superTblInfo); - } } if (retLen > remainderBufLen) { @@ -4551,21 +4552,22 @@ static int generateDataTail(char *tableName, int32_t tableSeq, int lenOfBinary = g_args.len_of_binary; int rand_num = taosRandom() % 100; + int randTail; + if ((g_args.disorderRatio != 0) && (rand_num < g_args.disorderRatio)) { - - int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k - - taosRandom() % g_args.disorderRange; - - retLen = generateData(data, data_type, - ncols_per_record, d, lenOfBinary); + randTail = (DEFAULT_TIMESTAMP_STEP * k + + (taosRandom() % g_args.disorderRange + 1)) * (-1); + debugPrint("rand data generated, back %d\n", randTail); } else { - retLen = generateData(data, data_type, - ncols_per_record, - startTime + DEFAULT_TIMESTAMP_STEP * k, - lenOfBinary); + randTail = DEFAULT_TIMESTAMP_STEP * k; } + retLen = generateData(data, data_type, + ncols_per_record, + startTime + randTail, + lenOfBinary); + if (len > remainderBufLen) break; @@ -5106,7 +5108,7 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { - int64_t d = winfo->lastTs - taosRandom() % winfo->superTblInfo->disorderRange; + int64_t d = winfo->lastTs - (taosRandom() % winfo->superTblInfo->disorderRange + 1); generateRowData(data, d, winfo->superTblInfo); } else { generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo); From 807f9b88750e755856708cd80bca2a1826f03ab2 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 9 Apr 2021 15:24:47 +0800 Subject: [PATCH 14/32] fix bug --- src/query/src/qResultbuf.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index c5dd6b3cac..f83caf2d8f 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -287,6 +287,10 @@ static void lruListMoveToFront(SList *pList, SPageInfo* pi) { tdListPrependNode(pList, pi->pn); } +static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { + return pageSize + POINTER_BYTES + 2 + sizeof(tFilePage); +} + tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { pResultBuf->statis.getPages += 1; @@ -311,7 +315,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 // allocate buf if (availablePage == NULL) { - pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES + 2); // add extract bytes in case of zipped buffer increased. + pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased. } else { pi->pData = availablePage; } @@ -355,7 +359,7 @@ tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { } if (availablePage == NULL) { - (*pi)->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES); + (*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); } else { (*pi)->pData = availablePage; } From 9b0af30e3c732f145d62420db4f8e84761540f93 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Fri, 9 Apr 2021 15:58:57 +0800 Subject: [PATCH 15/32] fix bug --- src/client/inc/tsclient.h | 2 +- src/client/src/tscSql.c | 7 ++++--- src/client/src/tscUtil.c | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 4869f65645..2a78de0ba4 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -125,7 +125,6 @@ typedef struct SInternalField { typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result - TAOS_FIELD* final; SArray *internalField; // SArray } SFieldInfo; @@ -316,6 +315,7 @@ typedef struct { char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex* pColumnIndex; + TAOS_FIELD* final; SArithmeticSupport *pArithSup; // support the arithmetic expression calculation on agg functions struct SLocalMerger *pLocalMerger; } SSqlRes; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 93d0e9fd09..13c8f025ea 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -405,6 +405,7 @@ int taos_affected_rows(TAOS_RES *tres) { TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; + SSqlRes *pRes = &pSql->res; if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); @@ -419,7 +420,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { SFieldInfo *pFieldInfo = &pQueryInfo->fieldsInfo; - if (pFieldInfo->final == NULL) { + if (pRes->final == NULL) { TAOS_FIELD* f = calloc(pFieldInfo->numOfOutput, sizeof(TAOS_FIELD)); int32_t j = 0; @@ -439,10 +440,10 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { } } - pFieldInfo->final = f; + pRes->final = f; } - return pFieldInfo->final; + return pRes->final; } static bool needToFetchNewBlock(SSqlObj* pSql) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 438e5618df..7d6d74b425 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -429,6 +429,8 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { tfree(pRes->pArithSup->data); tfree(pRes->pArithSup); } + + tfree(pRes->final); pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } @@ -1153,7 +1155,6 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } taosArrayDestroy(pFieldInfo->internalField); - tfree(pFieldInfo->final); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } From cfba07e788d5ded79a8bba47b066cb972a822443 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 9 Apr 2021 16:14:08 +0800 Subject: [PATCH 16/32] [TD-3722]: taosdemo performance regression due to always set random seed. (#5740) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index e804d93619..65ba7d50fe 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -507,11 +507,6 @@ static void resetAfterAnsiEscape(void) { static int taosRandom() { - struct timeval tv; - - gettimeofday(&tv, NULL); - srand(tv.tv_usec); - return rand(); } From e39139d1c033f41739aafd3f4f35645a548bdae1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 9 Apr 2021 15:46:28 +0800 Subject: [PATCH 17/32] [TD-3706]: [mnode] validate super table columns and tags count --- src/inc/taoserror.h | 1 + src/mnode/src/mnodeTable.c | 28 ++++++++++++++++++++-------- src/util/src/terror.c | 1 + 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 8daccee1f2..eff4eecbc1 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -163,6 +163,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") #define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") #define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) //"Too many tags") +#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x0365) //"Too many columns") #define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) //"Too many time series") #define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) //"Not super table") // operation only available for super table #define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) //"Tag name too long") diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 39eca8819d..2a8e941fcb 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1037,6 +1037,19 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { SCreateTableMsg* pCreate = (SCreateTableMsg*)((char*)pCreate1 + sizeof(SCMCreateTableMsg)); + int16_t numOfTags = htons(pCreate->numOfTags); + if (numOfTags > TSDB_MAX_TAGS) { + mError("msg:%p, app:%p table:%s, failed to create, too many tags", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); + return TSDB_CODE_MND_TOO_MANY_TAGS; + } + + int16_t numOfColumns = htons(pCreate->numOfColumns); + int32_t numOfCols = numOfColumns + numOfTags; + if (numOfCols > TSDB_MAX_COLUMNS) { + mError("msg:%p, app:%p table:%s, failed to create, too many columns", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); + return TSDB_CODE_MND_TOO_MANY_COLUMNS; + } + SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); @@ -1050,10 +1063,9 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); pStable->sversion = 0; pStable->tversion = 0; - pStable->numOfColumns = htons(pCreate->numOfColumns); - pStable->numOfTags = htons(pCreate->numOfTags); + pStable->numOfColumns = numOfColumns; + pStable->numOfTags = numOfTags; - int32_t numOfCols = pStable->numOfColumns + pStable->numOfTags; int32_t schemaSize = numOfCols * sizeof(SSchema); pStable->schema = (SSchema *)calloc(1, schemaSize); if (pStable->schema == NULL) { @@ -1064,11 +1076,6 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); - if (pStable->numOfColumns > TSDB_MAX_COLUMNS || pStable->numOfTags > TSDB_MAX_TAGS) { - mError("msg:%p, app:%p table:%s, failed to create, too many columns", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); - return TSDB_CODE_MND_INVALID_TABLE_NAME; - } - pStable->nextColId = 0; for (int32_t col = 0; col < numOfCols; col++) { @@ -1340,6 +1347,11 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 return TSDB_CODE_MND_APP_ERROR; } + if (pStable->numOfColumns + ncols + pStable->numOfTags > TSDB_MAX_COLUMNS) { + mError("msg:%p, app:%p stable:%s, add column, too many columns", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId); + return TSDB_CODE_MND_TOO_MANY_COLUMNS; + } + for (int32_t i = 0; i < ncols; i++) { if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add column, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 918ccc4935..586a886f47 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -175,6 +175,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") // operation only available for super table TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") From 824b3726f995076b01fe07d501c292487ae11c98 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 9 Apr 2021 08:52:29 +0000 Subject: [PATCH 18/32] [TD-3606]: When adding a new node to the cluster, the locale and charset are no longer checked --- src/mnode/src/mnodeDnode.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 80473ba5ae..85d9f94b88 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -437,14 +437,14 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { return TAOS_DN_OFF_TIME_ZONE_NOT_MATCH; } - if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) { - mError("\"locale\"[%s - %s] cfg parameters inconsistent", clusterCfg->locale, tsLocale); - return TAOS_DN_OFF_LOCALE_NOT_MATCH; - } - if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) { - mError("\"charset\"[%s - %s] cfg parameters inconsistent.", clusterCfg->charset, tsCharset); - return TAOS_DN_OFF_CHARSET_NOT_MATCH; - } + // if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) { + // mError("\"locale\"[%s - %s] cfg parameters inconsistent", clusterCfg->locale, tsLocale); + // return TAOS_DN_OFF_LOCALE_NOT_MATCH; + // } + // if (0 != strncasecmp(clusterCfg->charset, tsCharset, strlen(tsCharset))) { + // mError("\"charset\"[%s - %s] cfg parameters inconsistent.", clusterCfg->charset, tsCharset); + // return TAOS_DN_OFF_CHARSET_NOT_MATCH; + // } if (clusterCfg->enableBalance != tsEnableBalance) { mError("\"balance\"[%d - %d] cfg parameters inconsistent", clusterCfg->enableBalance, tsEnableBalance); From f4a0c1d97f6418ea01b0cd1630d699b06c2c4255 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 9 Apr 2021 18:37:40 +0800 Subject: [PATCH 19/32] [TD-3727]Suppressing errors in valgrind --- tests/pytest/crash_gen/valgrind_taos.supp | 85 +++++++++++++++++++++++ tests/pytest/handle_crash_gen_val_log.sh | 7 +- tests/pytest/handle_taosd_val_log.sh | 7 +- 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index 123858b3db..5eb5403395 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -17247,3 +17247,88 @@ fun:_PyEval_EvalFrameDefault fun:_PyEval_EvalCodeWithName } +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:_PyEval_EvalCodeWithName + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName + fun:PyEval_EvalCode + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyFunction_Vectorcall + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + obj:/usr/bin/python3.8 + fun:_PyEval_EvalFrameDefault + fun:_PyEval_EvalCodeWithName +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + obj:/usr/lib/python3/dist-packages/_cffi_backend.cpython-38-x86_64-linux-gnu.so + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyObject_CallMethod + fun:PyInit__openssl + fun:_PyImport_LoadDynamicModuleWithSpec + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 + fun:PyVectorcall_Call + fun:_PyEval_EvalFrameDefault +} diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index 55c10639d7..01cc62aaf8 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -36,16 +36,11 @@ for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | do defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then + if [ "$defiMemError" -gt 0 ]; then cat valgrind.err echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" exit 8 - elif [ "$defiMemError" -gt 1013 ];then #add for azure - cat valgrind.err - echo -e "${RED} ## Memory errors number valgrind reports \ - Definitely lost is $defiMemError. More than our threshold! ## ${NC}" - exit 8 fi fi done diff --git a/tests/pytest/handle_taosd_val_log.sh b/tests/pytest/handle_taosd_val_log.sh index 3161b5ff89..829bc68225 100755 --- a/tests/pytest/handle_taosd_val_log.sh +++ b/tests/pytest/handle_taosd_val_log.sh @@ -53,16 +53,11 @@ for defiMemError in `grep 'definitely lost:' taosd-definitely-lost-out.log | awk do defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then + if [ "$defiMemError" -gt 0 ]; then cat $VALGRIND_ERR echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" exit 8 - elif [ "$defiMemError" -gt 1013 ];then #add for azure - cat $VALGRIND_ERR - echo -e "${RED} ## Memory errors number valgrind reports \ - Definitely lost is $defiMemError. More than our threshold! ## ${NC}" - exit 8 fi fi done From 898cc038e754dab7df191f7d5255f7a77b242413 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 9 Apr 2021 18:39:48 +0800 Subject: [PATCH 20/32] [TD-3681]: fix syncdb precondition checkings --- src/mnode/src/mnodeVgroup.c | 1 + src/sync/src/syncMain.c | 10 ++++------ src/vnode/src/vnodeMain.c | 7 ++++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7eb3122d83..7222c8d1a0 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -994,6 +994,7 @@ void mnodeSendSyncVgroupMsg(SVgObj *pVgroup) { mDebug("vgId:%d, send sync all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, pVgroup->dbName); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + if (pVgroup->vnodeGid[i].role != TAOS_SYNC_ROLE_SLAVE) continue; SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, index:%d, send sync vnode msg to dnode %s", pVgroup->vgId, i, pVgroup->vnodeGid[i].pDnode->dnodeEp); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 29ced21291..2d33f68af0 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -409,23 +409,22 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force) syncReleaseNode(pNode); } -#if 1 void syncRecover(int64_t rid) { SSyncPeer *pPeer; SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; - // to do: add a few lines to check if recover is OK - // if take this node to unsync state, the whole system may not work - nodeRole = TAOS_SYNC_ROLE_UNSYNCED; (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); - nodeVersion = 0; pthread_mutex_lock(&pNode->mutex); + nodeVersion = 0; + for (int32_t i = 0; i < pNode->replica; ++i) { + if (i == pNode->selfIndex) continue; + pPeer = pNode->peerInfo[i]; if (pPeer->peerFd >= 0) { syncRestartConnection(pPeer); @@ -436,7 +435,6 @@ void syncRecover(int64_t rid) { syncReleaseNode(pNode); } -#endif int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { SSyncNode *pNode = syncAcquireNode(rid); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 69d94aff87..ee8ed9e0fa 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -99,8 +99,13 @@ int32_t vnodeSync(int32_t vgId) { return TSDB_CODE_VND_INVALID_VGROUP_ID; } - if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { + if (pVnode->role == TAOS_SYNC_ROLE_SLAVE) { vInfo("vgId:%d, vnode will sync, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + + pVnode->version = 0; + pVnode->fversion = 0; + walResetVersion(pVnode->wal, pVnode->fversion); + syncRecover(pVnode->sync); } From defd55e58777b601f8f23ac00fb9c7a1f4af6858 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 9 Apr 2021 18:50:29 +0800 Subject: [PATCH 21/32] [TD-3705]: taosdemo columns or tags count overflow. (#5745) * [TD-3705]: taosdemo columns or tags count overflow. * [TD-3705]: taosdemo columns or tags count overflow. compare columns+tags with max columns count. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 124 +++++++++++++++++++++++------------- 1 file changed, 78 insertions(+), 46 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 65ba7d50fe..49a3297bf6 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2364,7 +2364,8 @@ static int createSuperTable(TAOS * taos, char* dbName, lenOfOneRow += 21; } else { taos_close(taos); - printf("config error data type : %s\n", dataType); + errorPrint("%s() LN%d, config error data type : %s\n", + __func__, __LINE__, dataType); exit(-1); } } @@ -2382,7 +2383,8 @@ static int createSuperTable(TAOS * taos, char* dbName, } snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); - verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbl->colsOfCreateChildTable); + verbosePrint("%s() LN%d: %s\n", + __func__, __LINE__, superTbl->colsOfCreateChildTable); if (superTbl->tagCount == 0) { errorPrint("%s() LN%d, super table tag count is %d\n", @@ -2437,7 +2439,8 @@ static int createSuperTable(TAOS * taos, char* dbName, lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; } else { taos_close(taos); - printf("config error tag type : %s\n", dataType); + errorPrint("%s() LN%d, config error tag type : %s\n", + __func__, __LINE__, dataType); exit(-1); } } @@ -2732,7 +2735,8 @@ static int startMultiThreadCreateChildTable( db_name, g_Dbs.port); if (t_info->taos == NULL) { - errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n", + __func__, __LINE__, taos_errstr(NULL)); free(pids); free(infos); return -1; @@ -2793,35 +2797,35 @@ static void createChildTables() { } } } else { - // normal table - len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); - int j = 0; - while(g_args.datatype[j]) { - if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) - || (strncasecmp(g_args.datatype[j], - "NCHAR", strlen("NCHAR")) == 0)) { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s(60)", j, g_args.datatype[j]); - } else { - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, - ", COL%d %s", j, g_args.datatype[j]); - } - len = strlen(tblColsBuf); - j++; - } + // normal table + len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + int j = 0; + while(g_args.datatype[j]) { + if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) + || (strncasecmp(g_args.datatype[j], + "NCHAR", strlen("NCHAR")) == 0)) { + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + ", COL%d %s(60)", j, g_args.datatype[j]); + } else { + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + ", COL%d %s", j, g_args.datatype[j]); + } + len = strlen(tblColsBuf); + j++; + } - snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); - verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n", - __func__, __LINE__, - g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); - startMultiThreadCreateChildTable( - tblColsBuf, - g_Dbs.threadCountByCreateTbl, - 0, - g_args.num_of_tables, - g_Dbs.db[i].dbName, - NULL); + verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n", + __func__, __LINE__, + g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); + startMultiThreadCreateChildTable( + tblColsBuf, + g_Dbs.threadCountByCreateTbl, + 0, + g_args.num_of_tables, + g_Dbs.db[i].dbName, + NULL); } } } @@ -3035,6 +3039,13 @@ static bool getColumnAndTagTypeFromInsertJsonFile( index++; } } + + if (index > MAX_COLUMN_COUNT) { + errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", + __func__, __LINE__, MAX_COLUMN_COUNT); + goto PARSE_OVER; + } + superTbls->columnCount = index; count = 1; @@ -3099,8 +3110,20 @@ static bool getColumnAndTagTypeFromInsertJsonFile( index++; } } + + if (index > MAX_TAG_COUNT) { + errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", + __func__, __LINE__, MAX_TAG_COUNT); + goto PARSE_OVER; + } + superTbls->tagCount = index; + if ((superTbls->columnCount + superTbls->tagCount) > MAX_COLUMN_COUNT) { + errorPrint("%s() LN%d, columns + tags is more than max columns count: %d\n", + __func__, __LINE__, MAX_TAG_COUNT); + goto PARSE_OVER; + } ret = true; PARSE_OVER: @@ -4324,13 +4347,20 @@ static int generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo "%f, ", rand_double()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_bool()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { - dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "tinyint", strlen("tinyint"))) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "bool", strlen("bool"))) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "timestamp", strlen("timestamp"))) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%"PRId64", ", rand_bigint()); } else { errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType); return -1; @@ -4422,7 +4452,8 @@ static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { int ret = readSampleFromCsvFileToMem(superTblInfo); if (0 != ret) { - errorPrint("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__); + errorPrint("%s() LN%d, read sample from csv file failed.\n", + __func__, __LINE__); tmfree(sampleDataBuf); superTblInfo->sampleDataBuf = NULL; return -1; @@ -4444,7 +4475,8 @@ static int execInsert(threadInfo *pThreadInfo, char *buffer, int k) } else { if (0 != postProceSql(g_Dbs.host, g_Dbs.port, buffer)) { affectedRows = -1; - printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID); + printf("========restful return fail, threadID[%d]\n", + pThreadInfo->threadID); } else { affectedRows = k; } @@ -4601,7 +4633,8 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, tableSeq % superTblInfo->tagSampleCount); } if (NULL == tagsValBuf) { - errorPrint("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); + errorPrint("%s() LN%d, tag buf failed to allocate memory\n", + __func__, __LINE__); return -1; } @@ -4698,11 +4731,11 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { // TODO: prompt tbl count multple interlace rows and batch // - char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1); + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + char* buffer = calloc(maxSqlLen, 1); if (NULL == buffer) { - errorPrint( "Failed to alloc %d Bytes, reason:%s\n", - superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, - strerror(errno)); + errorPrint( "%s() LN%d, Failed to alloc %d Bytes, reason:%s\n", + __func__, __LINE__, maxSqlLen, strerror(errno)); return NULL; } @@ -4751,7 +4784,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { bool flagSleep = true; int sleepTimeTotal = 0; - int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; int remainderBufLen; while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { From 677c7dcd931c051c5ccc67ca3c48411c369374a6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 9 Apr 2021 21:22:04 +0800 Subject: [PATCH 22/32] [TD-3715]: taosdemo interlace rows must less than insert rows. (#5743) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 49a3297bf6..c6d5933dc7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -4718,8 +4718,13 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__); SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; + if (interlaceRows > insertRows) + interlaceRows = insertRows; + int insertMode; if (interlaceRows > 0) { @@ -4746,7 +4751,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; - int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; uint64_t st = 0; From 00904c14078c22f6a77b3d40ccec35bb5ac176d0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 10 Apr 2021 01:41:49 +0800 Subject: [PATCH 23/32] remove invalid qId --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 133ae6d0ab..cd6b3cf80c 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1367,7 +1367,7 @@ static void rpcProcessConnError(void *param, void *id) { tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); - if (pContext->numOfTry >= pContext->epSet.numOfEps) { + if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TSDB_MSG_TYPE_FETCH) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; From a7f44f38e0be1da90cd8a541b9b34f0f11672101 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 10 Apr 2021 11:37:10 +0800 Subject: [PATCH 24/32] use global uid as query id --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeCfg.c | 2 ++ src/inc/query.h | 1 + src/query/src/qExecutor.c | 28 ++++++++++++++++++++++++++-- src/query/src/queryMain.c | 1 + src/vnode/src/vnodeRead.c | 3 ++- 7 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 3f96466cc0..26475834d5 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -39,6 +39,7 @@ extern int8_t tsEnableTelemetryReporting; extern char tsEmail[]; extern char tsArbitrator[]; extern int8_t tsArbOnline; +extern int32_t tsDnodeId; // common extern int tsRpcTimer; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index ea5bf7954d..69b01e6c08 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -43,6 +43,7 @@ int8_t tsEnableVnodeBak = 1; int8_t tsEnableTelemetryReporting = 1; int8_t tsArbOnline = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; +int32_t tsDnodeId = 0; // common int32_t tsRpcTimer = 1000; diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index fd5956b37f..c573d709f5 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -17,6 +17,7 @@ #include "os.h" #include "cJSON.h" #include "dnodeCfg.h" +#include "tglobal.h" static SDnodeCfg tsCfg = {0}; static pthread_mutex_t tsCfgMutex; @@ -70,6 +71,7 @@ static void dnodeResetCfg(SDnodeCfg *cfg) { pthread_mutex_lock(&tsCfgMutex); tsCfg.dnodeId = cfg->dnodeId; + tsDnodeId = cfg->dnodeId; tstrncpy(tsCfg.clusterId, cfg->clusterId, TSDB_CLUSTER_ID_LEN); dnodePrintCfg(cfg); dnodeWriteCfg(); diff --git a/src/inc/query.h b/src/inc/query.h index c9dabcef54..d4ff811a8d 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -92,6 +92,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qId, uint64_t qInfo); void** qAcquireQInfo(void* pMgmt, uint64_t key); void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle); bool checkQIdEqual(void *qHandle, uint64_t qId); +int64_t genQueryId(void); #ifdef __cplusplus } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index dda67d77b2..c7e4223db5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -105,6 +105,30 @@ int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } + +int64_t genQueryId(void) { + int64_t uid = 0; + int64_t did = tsDnodeId; + + uid = did << 54; + + int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; + + uid |= pid << 44; + + int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; + + uid |= ts << 11; + + int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; + + uid |= sid; + + return uid; +} + + + static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->interval.intervalUnit != 'n' && pQuery->interval.intervalUnit != 'y') { @@ -6184,6 +6208,8 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SSqlGroupbyExpr* pGroupbyExpr goto _cleanup_qinfo; } + pQInfo->qId = *qId; + // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; SQuery* pQuery = &pQInfo->query; @@ -6316,8 +6342,6 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SSqlGroupbyExpr* pGroupbyExpr // todo refactor pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); - pQInfo->qId = atomic_add_fetch_64(&queryHandleId, 1); - *qId = pQInfo->qId; qDebug("qmsg:%p QInfo:%" PRIu64 "-%p created", pQueryMsg, pQInfo->qId, pQInfo); return pQInfo; diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 2ff0c9676e..838f6aa0c9 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -197,6 +197,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi return code; } + bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { SQInfo *pQInfo = (SQInfo *)qinfo; assert(pQInfo && pQInfo->signature == pQInfo); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 0836ade77f..5a7fc52931 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -208,6 +208,7 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) { pRsp->completed = true; } + static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { void * pCont = pRead->pCont; int32_t contLen = pRead->contLen; @@ -226,7 +227,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { if (contLen != 0) { qinfo_t pQInfo = NULL; - uint64_t qId = 0; + uint64_t qId = genQueryId(); code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, &qId); SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); From cdbb75609f87fbbd364d994faef8f30c922c214c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 10 Apr 2021 13:40:27 +0800 Subject: [PATCH 25/32] Feature/sangshuduo/td 3735 make test framework work on mac (#5761) (#5763) * [TD-3735]: make test framework work on mac * [TD-3735]: make test framework work on mac change cut fields to -f for mac compatibility. --- tests/test-all.sh | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index 3c8aed7d18..980629e6d2 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -80,6 +80,7 @@ function runSimCaseOneByOne { fi done < $1 } + function runSimCaseOneByOnefq { start=`sed -n "/$1-start/=" jenkins/basic.txt` @@ -155,6 +156,7 @@ function runPyCaseOneByOne { fi done < $1 } + function runPyCaseOneByOnefq() { cd $tests_dir/pytest if [[ $1 =~ full ]] ; then @@ -202,13 +204,37 @@ function runPyCaseOneByOnefq() { rm -rf ../../sim/case.log } +###################### +# main entry +###################### + +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) OS=Linux;; + Darwin*) OS=Darwin;; + CYGWIN*) OS=Windows;; + *) OS=Unknown;; +esac + +case "${OS}" in + Linux*) TAOSLIB=libtaos.so;; + Darwin*) TAOSLIB=libtaos.dylib;; + Windows*) TAOSLIB=taos.dll;; + Unknown) TAOSLIB="UNKNOWN:${unameOut}";; +esac + +echo TAOSLIB is ${TAOSLIB} + totalFailed=0 totalPyFailed=0 totalJDBCFailed=0 totalUnitFailed=0 totalExampleFailed=0 -corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` +if [ "${OS}" == "Linux" ]; then + corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` +fi + if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != "example" ]; then echo "### run TSIM test case ###" cd $tests_dir/script @@ -290,11 +316,11 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != " fi TOP_DIR=`pwd` - TAOSLIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1` + TAOSLIB_DIR=`find . -name "${TAOSLIB}"|grep -w lib|head -n1` if [[ "$TAOSLIB_DIR" == *"$IN_TDINTERNAL"* ]]; then - LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4,5` + LIB_DIR=`find . -name "${TAOSLIB}"|grep -w lib|head -n1|cut -d '/' -f 2,3,4,5` else - LIB_DIR=`find . -name "libtaos.so"|grep -w lib|head -n1|cut -d '/' --fields=2,3,4` + LIB_DIR=`find . -name "${TAOSLIB}"|grep -w lib|head -n1|cut -d '/' -f 2,3,4` fi export LD_LIBRARY_PATH=$TOP_DIR/$LIB_DIR:$LD_LIBRARY_PATH @@ -499,7 +525,9 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != echo -e "\n${RED} ### Total $totalExampleFailed examples failed! ### ${NC}" fi - dohavecore 1 + if [ "${OS}" == "Linux" ]; then + dohavecore 1 + fi fi From d3999c7bf3fed26df015bf541386c1df5a81a152 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 10 Apr 2021 14:01:29 +0800 Subject: [PATCH 26/32] fix bug --- src/cq/src/cqMain.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index f620eb4dd5..d4d202267c 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -73,6 +73,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); int32_t cqObjRef = -1; +int32_t cqVnodeNum = 0; void cqRmFromList(SCqObj *pObj) { //LOCK in caller @@ -166,6 +167,8 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { return NULL; } + atomic_add_fetch_32(&cqVnodeNum, 1); + cqCreateRef(); pContext->tmrCtrl = taosTmrInit(0, 0, 0, "CQ"); @@ -240,6 +243,13 @@ void cqClose(void *handle) { if (hasCq == 0) { freeSCqContext(pContext); } + + int32_t remainn = atomic_sub_fetch_32(&cqVnodeNum, 1); + if (remainn <= 0) { + int32_t ref = cqObjRef; + cqObjRef = -1; + taosCloseRef(ref); + } } void cqStart(void *handle) { From 483574a09cad33e7ce67b9ab0637c5abad59a4b2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 10 Apr 2021 17:24:43 +0800 Subject: [PATCH 27/32] [TD-3733]: taosdemo buffer processing refactor. (#5766) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 123 ++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 46 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index c6d5933dc7..c34b00fc8c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2949,9 +2949,6 @@ static int readSampleFromCsvFileToMem( continue; } - verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen, - superTblInfo->lenOfOneRow, getRows); - memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); getRows++; @@ -3527,6 +3524,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } + /* cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; @@ -3536,6 +3534,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; } + */ cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no if (childTblExists @@ -4619,7 +4618,8 @@ static int generateDataTail(char *tableName, int32_t tableSeq, } static int generateSQLHead(char *tableName, int32_t tableSeq, - threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer) + threadInfo* pThreadInfo, SSuperTable* superTblInfo, + char *buffer, int remainderBufLen) { int len; if (superTblInfo) { @@ -4671,7 +4671,62 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, return len; } -static int generateProgressiveDataBuffer(char *pTblName, +static int generateInterlaceDataBuffer( + char *tableName, int batchPerTbl, int i, int batchPerTblTimes, + int32_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + int64_t startTime, + int *pRemainderBufLen) +{ + char *pstr = buffer; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + + int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, + superTblInfo, pstr, *pRemainderBufLen); + + if (headLen <= 0) { + return 0; + } + // generate data buffer + verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, i, buffer); + + pstr += headLen; + *pRemainderBufLen -= headLen; + + int dataLen = 0; + + verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", + pThreadInfo->threadID, __func__, __LINE__, + i, batchPerTblTimes, batchPerTbl); + + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } + } else { + startTime = 1500000000000; + } + int k = generateDataTail( + tableName, tableSeq, pThreadInfo, superTblInfo, + batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0, + startTime, + &(pThreadInfo->samplePos), &dataLen); + + if (k > 0) { + pstr += dataLen; + *pRemainderBufLen -= dataLen; + } else { + pstr -= headLen; + pstr[0] = '\0'; + } + + return k; +} + +static int generateProgressiveDataBuffer( + char *tableName, int32_t tableSeq, threadInfo *pThreadInfo, char *buffer, int64_t insertRows, @@ -4691,6 +4746,7 @@ static int generateProgressiveDataBuffer(char *pTblName, assert(buffer != NULL); + int k = 0; int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; int remainderBufLen = maxSqlLen; @@ -4698,14 +4754,17 @@ static int generateProgressiveDataBuffer(char *pTblName, char *pstr = buffer; - int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo, - buffer); + int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo, + buffer, remainderBufLen); + + if (headLen <= 0) { + return 0; + } pstr += headLen; remainderBufLen -= headLen; - int k; int dataLen; - k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, + k = generateDataTail(tableName, tableSeq, pThreadInfo, superTblInfo, g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); @@ -4811,50 +4870,23 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { return NULL; } - int headLen; - if (i == 0) { - headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, - superTblInfo, pstr); - } else { - headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values", - pThreadInfo->db_name, - tableName); - } - - // generate data buffer - verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n", - pThreadInfo->threadID, __func__, __LINE__, i, buffer); - - pstr += headLen; - remainderBufLen -= headLen; - - int dataLen = 0; - - verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", - pThreadInfo->threadID, __func__, __LINE__, - i, batchPerTblTimes, batchPerTbl); - - if (superTblInfo) { - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - startTime = taosGetTimestamp(pThreadInfo->time_precision); - } - } else { - startTime = 1500000000000; - } - int generated = generateDataTail( - tableName, tableSeq, pThreadInfo, superTblInfo, - batchPerTbl, pstr, remainderBufLen, insertRows, 0, + int generated = generateInterlaceDataBuffer( + tableName, batchPerTbl, i, batchPerTblTimes, + tableSeq, + pThreadInfo, pstr, + insertRows, startTime, - &(pThreadInfo->samplePos), &dataLen); + &remainderBufLen); if (generated < 0) { debugPrint("[%d] %s() LN%d, generated data is %d\n", pThreadInfo->threadID, __func__, __LINE__, generated); goto free_and_statistics_interlace; + } else if (generated == 0) { + break; } - pstr += dataLen; - remainderBufLen -= dataLen; + tableSeq ++; recOfBatch += batchPerTbl; // startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; @@ -4862,7 +4894,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, batchPerTbl, recOfBatch); - tableSeq ++; if (insertMode == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table From 6d593fe80af7a8e3ffaf1b5569360efc2264268f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 10 Apr 2021 23:14:01 +0800 Subject: [PATCH 28/32] [td-3571]: enable update the vgroup info during super table query. --- src/client/src/tscServer.c | 47 ++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b8beed392e..f0b88bb113 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -34,6 +34,7 @@ int tscKeepConn[TSDB_SQL_MAX] = {0}; TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt); void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); void tscSaveSubscriptionProgress(void* sub); +static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } static int32_t getWaitingTimeInterval(int32_t count) { @@ -78,7 +79,8 @@ static void tscEpSetHtons(SRpcEpSet *s) { for (int32_t i = 0; i < s->numOfEps; i++) { s->port[i] = htons(s->port[i]); } -} +} + bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; @@ -118,12 +120,15 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { return; } - int32_t vgId = pTableMetaInfo->pTableMeta->vgId; + int32_t vgId = -1; if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { - assert(vgId == 0); - return; + vgId = extractSTableQueryVgroupId(pTableMetaInfo); + } else { + vgId = pTableMetaInfo->pTableMeta->vgId; } + assert(vgId > 0); + SNewVgroupInfo vgroupInfo = {.vgId = -1}; taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo, sizeof(SNewVgroupInfo)); assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); @@ -140,6 +145,27 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); } +int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { + assert(pTableMetaInfo != NULL); + + int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + vgId = pTableIdList->vgInfo.vgId; + } + + return vgId; +} + void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; if (pObj == NULL) return; @@ -515,21 +541,22 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t vgIndex = pTableMetaInfo->vgroupIndex; + int32_t vgId = -1; + if (pTableMetaInfo->pVgroupTables == NULL) { SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - - pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); - tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, qId:%" PRIu64, pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex, pSql->res.qId); + vgId = pVgroupInfo->vgroups[vgIndex].vgId; } else { int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(vgIndex >= 0 && vgIndex < numOfVgroups); SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - - pRetrieveMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId); - tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, qId:%" PRIu64, pSql, pTableIdList->vgInfo.vgId, vgIndex, pSql->res.qId); + vgId = pTableIdList->vgInfo.vgId; } + + pRetrieveMsg->header.vgId = htonl(vgId); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, qId:%" PRIu64, pSql, vgId, vgIndex, pSql->res.qId); } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); From 023e7959f4c7cce17cdc9e728a1362404037e8c3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 11 Apr 2021 11:13:33 +0800 Subject: [PATCH 29/32] [td-225]fix error by merge --- src/client/inc/tsclient.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 9072b6f349..c91943e232 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -146,17 +146,6 @@ typedef struct SInternalField { SExprFilter *pFieldFilters; } SInternalField; -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SArray *internalField; // SArray -} SFieldInfo; - -typedef struct SColumn { - SColumnIndex colIndex; - int32_t numOfFilters; - SColumnFilterInfo *filterInfo; -} SColumn; - typedef struct SCond { uint64_t uid; int32_t len; // length of tag query condition data From 5e9cd734c6987a3eabc9944c90752f9616b6dd95 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 10 Apr 2021 02:23:44 +0800 Subject: [PATCH 30/32] enlarge connection pool timeout --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index cd6b3cf80c..7205bafd7d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -295,7 +295,7 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } } else { - pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime); + pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime*30); if ( pRpc->pCache == NULL ) { tError("%s failed to init connection cache", pRpc->label); rpcClose(pRpc); @@ -470,7 +470,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { taosTmrStopA(&pConn->pTimer); // set the idle timer to monitor the activity - taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); + taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime*30, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); rpcSendMsgToPeer(pConn, msg, msgLen); // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured From 67fd99e75629bd82bfef62eb5f4f71f93c1629f4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 11 Apr 2021 23:04:48 +0800 Subject: [PATCH 31/32] [td-225] update the local cached epset in sqlObj. --- src/client/src/tscSchemaUtil.c | 3 ++- src/client/src/tscServer.c | 43 +++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 67352ca71c..2ea382132b 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -144,8 +144,9 @@ SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) { SNewVgroupInfo info = {0}; info.numOfEps = pVgroupMsg->numOfEps; info.vgId = pVgroupMsg->vgId; - info.inUse = 0; + info.inUse = 0; // 0 is the default value of inUse in case of multiple replica + assert(info.numOfEps >= 1 && info.vgId >= 1); for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) { tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN); info.ep[i].port = pVgroupMsg->epAddr[i].port; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cfdf2d75dd..8889e25177 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -113,8 +113,8 @@ static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgrou } } -static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { - SSqlCmd *pCmd = &pObj->cmd; +static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { + SSqlCmd *pCmd = &pSql->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return; @@ -143,6 +143,12 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); + + // Update the local cached epSet info cached by SqlObj + int32_t inUse = pSql->epSet.inUse; + tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); + tscDebug("%p update the epSet in SqlObj, in use before:%d, after:%d", pSql, inUse, pSql->epSet.inUse); + } int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { @@ -2007,7 +2013,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { (vgroupInfo.inUse < 0)) { // vgroup info exists, compare with it vgroupInfo = createNewVgroupInfo(&pMetaMsg->vgroup); taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); - tscDebug("add new VgroupInfo, vgId:%d, total:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); + tscDebug("add new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); } } @@ -2159,18 +2165,33 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { tscError("%p empty vgroup info", pSql); } else { for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { - //just init, no need to lock - SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; + // just init, no need to lock + SVgroupInfo *pVgroup = &pInfo->vgroupList->vgroups[j]; SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - pVgroups->vgId = htonl(vmsg->vgId); - pVgroups->numOfEps = vmsg->numOfEps; + vmsg->vgId = htonl(vmsg->vgId); + vmsg->numOfEps = vmsg->numOfEps; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); + } - assert(pVgroups->numOfEps >= 1 && pVgroups->vgId >= 1); + SNewVgroupInfo newVi = createNewVgroupInfo(vmsg); + pVgroup->numOfEps = newVi.numOfEps; + pVgroup->vgId = newVi.vgId; + for (int32_t k = 0; k < vmsg->numOfEps; ++k) { + pVgroup->epAddr[k].port = newVi.ep[k].port; + pVgroup->epAddr[k].fqdn = strndup(newVi.ep[k].fqdn, TSDB_FQDN_LEN); + } - for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { - pVgroups->epAddr[k].port = htons(vmsg->epAddr[k].port); - pVgroups->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, tListLen(vmsg->epAddr[k].fqdn)); + // check if current buffer contains the vgroup info. + // If not, add it + SNewVgroupInfo existVgroupInfo = {.inUse = -1}; + taosHashGetClone(tscVgroupMap, &newVi.vgId, sizeof(newVi.vgId), NULL, &existVgroupInfo, sizeof(SNewVgroupInfo)); + + if (((existVgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&existVgroupInfo, vmsg)) || + (existVgroupInfo.inUse < 0)) { // vgroup info exists, compare with it + taosHashPut(tscVgroupMap, &newVi.vgId, sizeof(newVi.vgId), &newVi, sizeof(newVi)); + tscDebug("add new VgroupInfo, vgId:%d, total cached:%d", newVi.vgId, (int32_t) taosHashGetSize(tscVgroupMap)); } } } From f5db431c19973e56a11c1e644e521634a7d83fb2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 10 Apr 2021 17:11:58 +0800 Subject: [PATCH 32/32] reset timeout --- src/rpc/src/rpcMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 7205bafd7d..cd6b3cf80c 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -295,7 +295,7 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } } else { - pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime*30); + pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime); if ( pRpc->pCache == NULL ) { tError("%s failed to init connection cache", pRpc->label); rpcClose(pRpc); @@ -470,7 +470,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { taosTmrStopA(&pConn->pTimer); // set the idle timer to monitor the activity - taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime*30, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); + taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); rpcSendMsgToPeer(pConn, msg, msgLen); // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured