From ca68c37dffb243c35a58b7bca670497654dde9fe Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 18 Aug 2021 17:31:20 +0800 Subject: [PATCH 01/13] [TD-6011]: where clause including 'bool' Keyword causes core dump --- src/common/src/tvariant.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 33dab51633..3f9e219983 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -31,12 +31,12 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { switch (token->type) { case TSDB_DATA_TYPE_BOOL: { - int32_t k = strncasecmp(token->z, "true", 4); - if (k == 0) { + if (strncasecmp(token->z, "true", 4) == 0) { pVar->i64 = TSDB_TRUE; - } else { - assert(strncasecmp(token->z, "false", 5) == 0); + } else if (strncasecmp(token->z, "false", 5) == 0) { pVar->i64 = TSDB_FALSE; + } else { + return; } break; From b0b3810ffb248b6abf38d85778e0da4801ed0a68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 19 Aug 2021 18:02:55 +0800 Subject: [PATCH 02/13] [td-6229] Send the query request in parallel if the number of subquery exceeds the threshold. --- src/client/src/tscSubquery.c | 57 +++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 422d07ccde..75307087b1 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -15,8 +15,9 @@ #define _GNU_SOURCE #include "os.h" - #include "texpr.h" + +#include "tsched.h" #include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" @@ -2423,6 +2424,26 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { return terrno; } +typedef struct SPair { + int32_t first; + int32_t second; +} SPair; + +static void doSendQueryReqs(SSchedMsg* pSchedMsg) { + SSqlObj* pSql = pSchedMsg->ahandle; + SPair* p = pSchedMsg->msg; + + for(int32_t i = p->first; i < p->second; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + SRetrieveSupport* pSupport = pSub->param; + + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); + tscProcessSql(pSub); + } + + tfree(p); +} + int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -2546,13 +2567,33 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { doCleanupSubqueries(pSql, i); return pRes->code; } - - for(int32_t j = 0; j < pState->numOfSub; ++j) { - SSqlObj* pSub = pSql->pSubs[j]; - SRetrieveSupport* pSupport = pSub->param; - - tscDebug("0x%"PRIx64" sub:%p launch subquery, orderOfSub:%d.", pSql->self, pSub, pSupport->subqueryIndex); - tscProcessSql(pSub); + + // concurrently sent the query requests. + const int32_t MAX_REQUEST_PER_TASK = 8; + + int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; + assert(numOfTasks >= 1); + + int32_t num = (pState->numOfSub/numOfTasks) + 1; + tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); + + for(int32_t j = 0; j < numOfTasks; ++j) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = doSendQueryReqs; + schedMsg.ahandle = (void*)pSql; + + schedMsg.thandle = NULL; + SPair* p = calloc(1, sizeof(SPair)); + p->first = j * num; + + if (j == numOfTasks - 1) { + p->second = pState->numOfSub; + } else { + p->second = (j + 1) * num; + } + + schedMsg.msg = p; + taosScheduleTask(tscQhandle, &schedMsg); } return TSDB_CODE_SUCCESS; From 0afe768d767b8f9e7db929abb73d5fe39b177848 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 18:20:57 +0800 Subject: [PATCH 03/13] [TD-6228] fix tag condtion is too long error --- src/client/src/tscServer.c | 8 ++++---- src/client/src/tscUtil.c | 16 ++++++++++++++++ src/inc/taosmsg.h | 2 +- src/query/src/qExecutor.c | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index f8c072a239..953914c504 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -621,7 +621,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - + int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo); size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); int32_t exprSize = (int32_t)(sizeof(SSqlFuncMsg) * numOfExprs * 2); @@ -643,8 +643,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { tableSerialize = totalTables * sizeof(STableIdInfo); } - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg, int32_t *succeed) { @@ -1099,7 +1099,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htons(pCond->len); + pQueryMsg->tagCondLen = htonl(pCond->len); memcpy(pMsg, pCond->cond, pCond->len); pMsg += pCond->len; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 773243891f..5e59f54c88 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2989,4 +2989,20 @@ STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { return p; } +int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo) { + int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList); + int32_t len = 0; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i); + for (int32_t j = 0; j < pCol->numOfFilters; ++j) { + SColumnFilterInfo *pColFilter = &pCol->filterInfo[j]; + len += sizeof(SColumnFilterInfo); + if (pColFilter->filterstr) { + len += (int32_t)pColFilter->len + 1; + } + } + } + return len; +} diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index d7ac7dd277..2d78210844 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -494,7 +494,7 @@ typedef struct { int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; SSessionWindow sw; // session window - uint16_t tagCondLen; // tag length in current query + uint32_t tagCondLen; // tag length in current query uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6b32508f07..f96d33f060 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6172,7 +6172,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); - pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->tagCondLen = htonl(pQueryMsg->tagCondLen); pQueryMsg->tsOffset = htonl(pQueryMsg->tsOffset); pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); From d837e47cb6e9ec24925d9d50789484436306176e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 18:47:37 +0800 Subject: [PATCH 04/13] [TD-6228] fix tag condtion is too long error --- src/client/inc/tscUtil.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 47b2865313..a26f03003e 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -301,6 +301,7 @@ uint32_t tscGetTableMetaMaxSize(); int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); +int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo); void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); From 7c7f89a3eb7176313c15450fe4a0b84e85f0e4c3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 Aug 2021 21:04:04 +0800 Subject: [PATCH 05/13] [TD-6228] fix tag condtion is too long error --- src/client/src/tscServer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 953914c504..599767803c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -643,6 +643,12 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { tableSerialize = totalTables * sizeof(STableIdInfo); } + + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; + if (pCond->len > 0) { + srcColListSize += pCond->len; + } + return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } From deb5279ab7be9fd81a6377160d9b38697649edc5 Mon Sep 17 00:00:00 2001 From: tomchon Date: Fri, 20 Aug 2021 10:37:07 +0800 Subject: [PATCH 06/13] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 84ef060196..24d451e8f7 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.13") + SET(TD_VER_NUMBER "2.0.20.15") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b08ae53ecd..2c19e83b47 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.13' +version: '2.0.20.15' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.13 + - usr/lib/libtaos.so.2.0.20.15 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From f9f96640731135e4952d4c87410b28b8d77eba32 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 20 Aug 2021 10:46:21 +0800 Subject: [PATCH 07/13] [TD-6234]: add unit test case for TD-6011 --- tests/pytest/query/filterNoKeyword.py | 83 +++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/pytest/query/filterNoKeyword.py diff --git a/tests/pytest/query/filterNoKeyword.py b/tests/pytest/query/filterNoKeyword.py new file mode 100644 index 0000000000..34d74efd82 --- /dev/null +++ b/tests/pytest/query/filterNoKeyword.py @@ -0,0 +1,83 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + print("======= Verify filter for bool, nchar and binary type =========") + tdLog.debug( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + tdSql.execute( + "create table st(ts timestamp, tbcol1 bool, tbcol2 binary(10), tbcol3 nchar(20), tbcol4 tinyint, tbcol5 smallint, tbcol6 int, tbcol7 bigint, tbcol8 float, tbcol9 double) tags(tagcol1 bool, tagcol2 binary(10), tagcol3 nchar(10))") + + tdSql.execute("create table st1 using st tags(true, 'table1', '水表')") + for i in range(1, 6): + tdSql.execute( + "insert into st1 values(%d, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d, %f, %f)" % + (self.ts + i, i % + 2, i, i, + i, i, i, i, 1.0, 1.0)) + + # =============Data type keywords cannot be used in filter==================== + # timestamp + tdSql.error("select * from st where timestamp = 1629417600") + + # bool + tdSql.error("select * from st where bool = false") + + #binary + tdSql.error("select * from st where binary = 'taosdata'") + + # nchar + tdSql.error("select * from st where nchar = '涛思数据'") + + # tinyint + tdSql.error("select * from st where tinyint = 127") + + # smallint + tdSql.error("select * from st where smallint = 32767") + + # int + tdSql.error("select * from st where INTEGER = 2147483647") + tdSql.error("select * from st where int = 2147483647") + + # bigint + tdSql.error("select * from st where bigint = 2147483647") + + # float + tdSql.error("select * from st where float = 3.4E38") + + # double + tdSql.error("select * from st where double = 1.7E308") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 8d5c67ee5ded0080b2780d8f958375d160f1a2f5 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 20 Aug 2021 19:17:02 +0800 Subject: [PATCH 08/13] [TD-6164]: fix deadlock with fopen's malloc --- src/os/src/linux/linuxEnv.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index e3eadbc94b..a08a5aeed6 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -44,14 +44,20 @@ char cmdline[1024]; char* taosGetCmdlineByPID(int pid) { sprintf(cmdline, "/proc/%d/cmdline", pid); - FILE* f = fopen(cmdline, "r"); - if (f) { - size_t size; - size = fread(cmdline, sizeof(char), 1024, f); - if (size > 0) { - if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0'; - } - fclose(f); + + int fd = open(cmdline, O_RDONLY); + if (fd >= 0) { + int n = read(fd, cmdline, sizeof(cmdline) - 1); + if (n < 0) n = 0; + + if (n > 0 && cmdline[n - 1] == '\n') --n; + + cmdline[n] = 0; + + close(fd); + } else { + cmdline[0] = 0; } + return cmdline; } From c3d19d52c9944947425f8cde0c9248787550f84c Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Fri, 20 Aug 2021 19:57:15 +0800 Subject: [PATCH 09/13] TD-6162 repaire error in demo.py (#7422) --- src/connector/python/examples/demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/python/examples/demo.py b/src/connector/python/examples/demo.py index 6c7c03f3e2..3bc09046f3 100644 --- a/src/connector/python/examples/demo.py +++ b/src/connector/python/examples/demo.py @@ -2,7 +2,7 @@ import taos conn = taos.connect(host='127.0.0.1', user='root', - passworkd='taodata', + password='taosdata', database='log') cursor = conn.cursor() From a8f1208dbd46eb6262bd19c52dc6fd0a6c5fa35b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 20 Aug 2021 22:48:23 +0800 Subject: [PATCH 10/13] [td-6229] enable the last_row cache while query condition exists. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d6af2e2ef1..3926564b50 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7283,7 +7283,7 @@ int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t i } pQuerySqlNode->pWhere = NULL; - if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI && (!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER))) { pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } From 567d0fe8da077e8653d27c8410673f4a75a1e59c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 24 Aug 2021 08:33:44 +0800 Subject: [PATCH 11/13] [TD-6264]:backport _block_dist() from 2.1 to 2.0 to fix core dump due to tbuf length limitation --- src/inc/tsdb.h | 7 +- src/query/src/qAggMain.c | 148 +++++++++++++------- src/query/src/qExecutor.c | 10 +- src/query/src/qUtil.c | 16 ++- src/tsdb/inc/tsdbCommit.h | 2 + src/tsdb/src/tsdbRead.c | 12 +- src/util/inc/tarray.h | 8 ++ src/util/src/tarray.c | 5 + tests/script/general/compute/block_dist.sim | 6 +- 9 files changed, 158 insertions(+), 56 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85ee9f0443..da33a2ba73 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -211,7 +211,7 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct SFileBlockInfo { - int32_t numOfRows; + int32_t numBlocksOfStep; } SFileBlockInfo; typedef struct { @@ -225,13 +225,18 @@ typedef struct { SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; +#define TSDB_BLOCK_DIST_STEP_ROWS 16 typedef struct { uint16_t rowSize; uint16_t numOfFiles; uint32_t numOfTables; uint64_t totalSize; + uint64_t totalRows; + int32_t maxRows; + int32_t minRows; int32_t firstSeekTimeUs; uint32_t numOfRowsInMemTable; + uint32_t numOfSmallBlocks; SArray *dataBlockInfos; } STableBlockDist; diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index b3996fb55a..b0f7decfc6 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4794,7 +4794,7 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (int16_t) pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); @@ -4802,51 +4802,85 @@ void blockInfo_func(SQLFunctionCtx* pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) { +static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); + pDist->numOfTables += pSrc->numOfTables; pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; + pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; pDist->numOfFiles += pSrc->numOfFiles; pDist->totalSize += pSrc->totalSize; + pDist->totalRows += pSrc->totalRows; - if (pDist->dataBlockInfos == NULL) { - pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo)); + if (pResInfo->hasResult == DATA_SET_FLAG) { + pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); + pDist->minRows = MIN(pDist->minRows, pSrc->minRows); + } else { + pDist->maxRows = pSrc->maxRows; + pDist->minRows = pSrc->minRows; + + int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; + if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++maxSteps; + } + pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(pDist->dataBlockInfos, maxSteps); } - taosArrayPushBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos)); + size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); + for (int32_t i = 0; i < steps; ++i) { + int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; + SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); + blockInfo->numBlocksOfStep += srcNumBlocks; + } } void block_func_merge(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - - STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); STableBlockDist info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - mergeTableBlockDist(pDist, &info); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + mergeTableBlockDist(pResInfo, &info); + + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; } -static int32_t doGetPercentile(const SArray* pArray, double rate) { - int32_t len = (int32_t)taosArrayGetSize(pArray); - if (len <= 0) { - return 0; +void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, + double* percents, int32_t* percentiles) { + if (totalBlocks == 0) { + for (int32_t i = 0; i < numOfPercents; ++i) { + percentiles[i] = 0; + } + return; } - assert(rate >= 0 && rate <= 1.0); - int idx = (int32_t)((len - 1) * rate); + SArray *blocksInfos = pTableBlockDist->dataBlockInfos; + size_t numSteps = taosArrayGetSize(blocksInfos); + size_t cumulativeBlocks = 0; - return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows; -} + int percentIndex = 0; + for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { + int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; + if (numStepBlocks == 0) continue; + cumulativeBlocks += numStepBlocks; -static int compareBlockInfo(const void *pLeft, const void *pRight) { - int32_t left = ((SFileBlockInfo *)pLeft)->numOfRows; - int32_t right = ((SFileBlockInfo *)pRight)->numOfRows; + while (percentIndex < numOfPercents) { + double blockRank = totalBlocks * percents[percentIndex]; + if (blockRank <= cumulativeBlocks) { + percentiles[percentIndex] = indexStep; + ++percentIndex; + } else { + break; + } + } + } - if (left > right) return 1; - if (left < right) return -1; - return 0; + for (int32_t i = 0; i < numOfPercents; ++i) { + percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; + } } void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { @@ -4854,41 +4888,56 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { return; } - int64_t min = INT64_MAX, max = INT64_MIN, avg = 0; - SArray* blockInfos= pTableBlockDist->dataBlockInfos; - int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); + SArray* blockInfos = pTableBlockDist->dataBlockInfos; + uint64_t totalRows = pTableBlockDist->totalRows; + size_t numSteps = taosArrayGetSize(blockInfos); + int64_t totalBlocks = 0; + int64_t min = -1, max = -1, avg = 0; - for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { + for (int32_t i = 0; i < numSteps; i++) { SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t rows = blockInfo->numOfRows; - - min = MIN(min, rows); - max = MAX(max, rows); - totalRows += rows; + int64_t blocks = blockInfo->numBlocksOfStep; + totalBlocks += blocks; } avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - taosArraySort(blockInfos, compareBlockInfo); + min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; + max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; + + double stdDev = 0; + if (totalBlocks > 0) { + double variance = 0; + for (int32_t i = 0; i < numSteps; i++) { + SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); + int64_t blocks = blockInfo->numBlocksOfStep; + int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); + variance += blocks * (rows - avg) * (rows - avg); + } + variance = variance / totalBlocks; + stdDev = sqrt(variance); + } + + double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; + int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); + getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); uint64_t totalLen = pTableBlockDist->totalSize; int32_t rowSize = pTableBlockDist->rowSize; - + int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; + double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; int sz = sprintf(result + VARSTR_HEADER_SIZE, "summary: \n\t " "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t " - "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]", - doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10), - doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30), - doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50), - doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70), - doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90), - doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99), - min, max, avg, 0.0, - totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows), - pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs); + "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " + "RowsInMem=[%d] \n\t", + percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], + percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], + min, max, avg, stdDev, + totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, + pTableBlockDist->numOfRowsInMemTable); varDataSetLen(result, sz); UNUSED(sz); } @@ -4897,9 +4946,14 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); - pDist->rowSize = (int16_t)pCtx->param[0].i64; + pDist->rowSize = (uint16_t)pCtx->param[0].i64; generateBlockDistResult(pDist, pCtx->pOutput); + if (pDist->dataBlockInfos != NULL) { + taosArrayDestroy(pDist->dataBlockInfos); + pDist->dataBlockInfos = NULL; + } + // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f96d33f060..b2e63707cb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4546,7 +4546,15 @@ static SSDataBlock* doBlockInfoScan(void* param) { STableBlockDist tableBlockDist = {0}; tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; - tableBlockDist.dataBlockInfos = taosArrayInit(512, sizeof(SFileBlockInfo)); + + int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; + if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++numRowSteps; + } + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; + tableBlockDist.minRows = INT_MAX; tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index aa793add84..ed069c6a91 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -581,7 +581,11 @@ void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { tbufWriteUint32(bw, pDist->numOfTables); tbufWriteUint16(bw, pDist->numOfFiles); tbufWriteUint64(bw, pDist->totalSize); + tbufWriteUint64(bw, pDist->totalRows); + tbufWriteInt32(bw, pDist->maxRows); + tbufWriteInt32(bw, pDist->minRows); tbufWriteUint32(bw, pDist->numOfRowsInMemTable); + tbufWriteUint32(bw, pDist->numOfSmallBlocks); tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); // compress the binary string @@ -616,13 +620,17 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi pDist->numOfTables = tbufReadUint32(&br); pDist->numOfFiles = tbufReadUint16(&br); pDist->totalSize = tbufReadUint64(&br); + pDist->totalRows = tbufReadUint64(&br); + pDist->maxRows = tbufReadInt32(&br); + pDist->minRows = tbufReadInt32(&br); pDist->numOfRowsInMemTable = tbufReadUint32(&br); - int64_t numOfBlocks = tbufReadUint64(&br); + pDist->numOfSmallBlocks = tbufReadUint32(&br); + int64_t numSteps = tbufReadUint64(&br); bool comp = tbufReadUint8(&br); uint32_t compLen = tbufReadUint32(&br); - size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo)); + size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); char* outputBuf = NULL; if (comp) { @@ -633,12 +641,12 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo)); + assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); } else { outputBuf = (char*) tbufReadBinary(&br, &originalLen); } - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo)); + pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); if (comp) { tfree(outputBuf); } diff --git a/src/tsdb/inc/tsdbCommit.h b/src/tsdb/inc/tsdbCommit.h index 5e740081d1..f1c3a91746 100644 --- a/src/tsdb/inc/tsdbCommit.h +++ b/src/tsdb/inc/tsdbCommit.h @@ -29,6 +29,8 @@ typedef struct { int64_t size; } SKVRecord; +#define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) + void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn); int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index be686fcffd..1babb01530 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2104,6 +2104,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; pTableBlockInfo->totalSize = 0; + pTableBlockInfo->totalRows = 0; STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); // find the start data block in file @@ -2121,6 +2122,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { @@ -2136,7 +2138,7 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); @@ -2177,7 +2179,13 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist pTableBlockInfo->totalSize += pBlock[j].len; int32_t numOfRows = pBlock[j].numOfRows; - taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows); + pTableBlockInfo->totalRows += numOfRows; + if (numOfRows > pTableBlockInfo->maxRows) pTableBlockInfo->maxRows = numOfRows; + if (numOfRows < pTableBlockInfo->minRows) pTableBlockInfo->minRows = numOfRows; + if (numOfRows < defaultRows) pTableBlockInfo->numOfSmallBlocks+=1; + int32_t stepIndex = (numOfRows-1)/TSDB_BLOCK_DIST_STEP_ROWS; + SFileBlockInfo *blockInfo = (SFileBlockInfo*)taosArrayGet(pTableBlockInfo->dataBlockInfos, stepIndex); + blockInfo->numBlocksOfStep++; } } } diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index f2e268c2d4..677be0051e 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -98,6 +98,14 @@ void* taosArrayGetLast(const SArray* pArray); */ size_t taosArrayGetSize(const SArray* pArray); +/** + * set the size of array + * @param pArray + * @param size size of the array + * @return + */ +void taosArraySetSize(SArray* pArray, size_t size); + /** * insert data into array * @param pArray diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index 4dde5dbba2..6fb4029a84 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -110,6 +110,11 @@ void* taosArrayGetLast(const SArray* pArray) { size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; } +void taosArraySetSize(SArray* pArray, size_t size) { + assert(size <= pArray->capacity); + pArray->size = size; +} + void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { if (pArray == NULL || pData == NULL) { return NULL; diff --git a/tests/script/general/compute/block_dist.sim b/tests/script/general/compute/block_dist.sim index 51cf903654..5343c1db28 100644 --- a/tests/script/general/compute/block_dist.sim +++ b/tests/script/general/compute/block_dist.sim @@ -84,6 +84,10 @@ if $rows != 1 then return -1 endi +print ============== TD-5998 +sql_error select _block_dist() from (select * from $nt) +sql_error select _block_dist() from (select * from $mt) + print =============== clear sql drop database $db sql show databases @@ -91,4 +95,4 @@ if $rows != 0 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 78da4fedceb3b965145e03e9f9c2c60b8ea4c888 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Aug 2021 18:40:26 +0800 Subject: [PATCH 12/13] minor changes --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3314798d6d..efdc644f2c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2440,7 +2440,7 @@ static void doSendQueryReqs(SSchedMsg* pSchedMsg) { SRetrieveSupport* pSupport = pSub->param; tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); - tscProcessSql(pSub); + tscBuildAndSendRequest(pSub, NULL); } tfree(p); From aad82de6fcd940d0409853fb8696a2b2004115df Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Aug 2021 19:32:20 +0800 Subject: [PATCH 13/13] TS-177 change default log len from 100 to 512 --- src/plugins/monitor/src/monMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 6e583fe0df..fea793fa86 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -34,7 +34,7 @@ #define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} #define SQL_LENGTH 1030 -#define LOG_LEN_STR 100 +#define LOG_LEN_STR 512 #define IP_LEN_STR TSDB_EP_LEN #define CHECK_INTERVAL 1000