From 0e535f77f75a38fe924a6695117ebba42dd21cda Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 28 Jun 2022 09:43:17 +0800 Subject: [PATCH 01/24] fix constant --- tests/pytest/util/constant.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index 807398f70f..5be1c603b3 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -84,16 +84,16 @@ SMALLINT_MIN = -32768 SMALLINT_UN_MAX = 65535 SMALLINT_UN_MIN = 0 -INT_MAX = 2147483647 -INT_MIN = -2147483648 +INT_MAX = 2_147_483_647 +INT_MIN = -2_147_483_648 -INT_UN_MAX = 4294967295 +INT_UN_MAX = 4_294_967_295 INT_UN_MIN = 0 -BIGINT_MAX = 9223372036854775807 -BIGINT_MIN = -9223372036854775808 +BIGINT_MAX = 9_223_372_036_854_775_807 +BIGINT_MIN = -9_223_372_036_854_775_808 -BIGINT_UN_MAX = 18446744073709551615 +BIGINT_UN_MAX = 18_446_744_073_709_551_615 BIGINT_UN_MIN = 0 FLOAT_MAX = 3.40E+38 @@ -131,13 +131,13 @@ COL_COUNT_MIN = 2 TAG_COL_COUNT_MAX = 4096 TAG_COL_COUNT_MIN = 3 -MNODE_SHM_SIZE_MAX = 2147483647 -MNODE_SHM_SIZE_MIN = 6292480 -MNODE_SHM_SIZE_DEFAULT = 6292480 +MNODE_SHM_SIZE_MAX = 2_147_483_647 +MNODE_SHM_SIZE_MIN = 6_292_480 +MNODE_SHM_SIZE_DEFAULT = 6_292_480 -VNODE_SHM_SIZE_MAX = 2147483647 -VNODE_SHM_SIZE_MIN = 6292480 -VNODE_SHM_SIZE_DEFAULT = 31458304 +VNODE_SHM_SIZE_MAX = 2_147_483_647 +VNODE_SHM_SIZE_MIN = 6_292_480 +VNODE_SHM_SIZE_DEFAULT = 31_458_304 # time_init TIME_MS = 1 @@ -160,6 +160,7 @@ INTERVAL_MIN = 1 * TIME_MS if PRECISION == PRECISION_DEFAULT else 1 * TIME_US # streams and related agg-function SMA_INDEX_FUNCTIONS = ["MIN", "MAX"] ROLLUP_FUNCTIONS = ["AVG", "SUM", "MIN", "MAX", "LAST", "FIRST"] +BLOCK_FUNCTIONS = ["SUM", "MIN", "MAX"] SMA_WATMARK_MAXDELAY_INIT = ['a', "s", "m"] WATERMARK_MAX = 900000 WATERMARK_MIN = 0 From b6d66571744f1b38725db17ce77a358b236f80d6 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Jun 2022 15:14:16 +0800 Subject: [PATCH 02/24] feat: support push agg operator condition to scan --- source/libs/nodes/src/nodesUtilFuncs.c | 1 - source/libs/planner/src/planOptimizer.c | 155 +++++++++++++++++++++++- 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index dc9d9b92ee..1972010e25 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1728,7 +1728,6 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) { } else { pCxt->hasOtherCol = true; } - return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; } return DEAL_RES_CONTINUE; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b5b53f00a0..a4641a4709 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -587,11 +587,160 @@ static int32_t pushDownCondOptDealJoin(SOptimizeContext* pCxt, SJoinLogicNode* p return code; } -static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { - // todo +typedef struct SPartAggCondContext { + SAggLogicNode* pAgg; + bool hasAggFunc; +} SPartAggCondContext; + +static EDealRes partAggCondHasAggFuncImpl(SNode* pNode, void* pContext) { + SPartAggCondContext* pCxt = pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SNode* pAggFunc = NULL; + FOREACH(pAggFunc, pCxt->pAgg->pAggFuncs) { + if (strcmp(((SColumnNode*)pNode)->colName, ((SFunctionNode*)pAggFunc)->node.aliasName) == 0) { + pCxt->hasAggFunc = true; + return DEAL_RES_END; + } + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t partitionAggCondHasAggFunc(SAggLogicNode* pAgg, SNode* pCond) { + SPartAggCondContext cxt = {.pAgg = pAgg, .hasAggFunc = false}; + nodesWalkExpr(pCond, partAggCondHasAggFuncImpl, &cxt); + return cxt.hasAggFunc; +} + +static int32_t partitionAggCondConj(SAggLogicNode* pAgg, SNode** ppAggFuncCond, SNode** ppGroupKeyCond) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pAgg->node.pConditions; + int32_t code = TSDB_CODE_SUCCESS; + + SNodeList* pAggFuncConds = NULL; + SNodeList* pGroupKeyConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (partitionAggCondHasAggFunc(pAgg, pCond)) { + code = nodesListMakeAppend(&pAggFuncConds, nodesCloneNode(pCond)); + } else { + code = nodesListMakeAppend(&pGroupKeyConds, nodesCloneNode(pCond)); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNode* pTempAggFuncCond = NULL; + SNode* pTempGroupKeyCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempAggFuncCond, &pAggFuncConds); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempGroupKeyCond, &pGroupKeyConds); + } + + if (TSDB_CODE_SUCCESS == code) { + *ppAggFuncCond = pTempAggFuncCond; + *ppGroupKeyCond = pTempGroupKeyCond; + } else { + nodesDestroyList(pAggFuncConds); + nodesDestroyList(pGroupKeyConds); + nodesDestroyNode(pTempAggFuncCond); + nodesDestroyNode(pTempGroupKeyCond); + } + pAgg->node.pConditions = NULL; + return code; +} + +static int32_t partitionAggCond(SAggLogicNode* pAgg, SNode** ppAggFunCond, SNode** ppGroupKeyCond) { + SNode* pAggNodeCond = pAgg->node.pConditions; + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pAggNodeCond) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)(pAggNodeCond))->condType) { + return partitionAggCondConj(pAgg, ppAggFunCond, ppGroupKeyCond); + } + if (partitionAggCondHasAggFunc(pAgg, pAggNodeCond)) { + *ppAggFunCond = pAggNodeCond; + } else { + *ppGroupKeyCond = pAggNodeCond; + } + pAgg->node.pConditions = NULL; return TSDB_CODE_SUCCESS; } +static int32_t pushCondToAggCond(SOptimizeContext* pCxt, SAggLogicNode* pAgg, SNode** pAggFuncCond) { + pushDownCondOptAppendCond(&pAgg->node.pConditions, pAggFuncCond); + return TSDB_CODE_SUCCESS; +} + +typedef struct SRewriteAggGroupKeyCondContext{ + SAggLogicNode *pAgg; + int32_t errCode; +} SRewriteAggGroupKeyCondContext; + +static EDealRes rewriteAggGroupKeyCondForPushDownImpl(SNode** pNode, void* pContext) { + SRewriteAggGroupKeyCondContext* pCxt = pContext; + SAggLogicNode* pAgg = pCxt->pAgg; + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, pAgg->pGroupKeys) { + SNode* pGroup = NULL; + FOREACH(pGroup, ((SGroupingSetNode*)pGroupKey)->pParameterList) { + if (0 == strcmp(((SExprNode*)pGroup)->aliasName, ((SColumnNode*)(*pNode))->colName)) { + SNode* pExpr = nodesCloneNode(pGroup); + if (pExpr == NULL) { + pCxt->errCode = terrno; + return DEAL_RES_ERROR; + } + nodesDestroyNode(*pNode); + *pNode = pExpr; + } + } + } + return DEAL_RES_IGNORE_CHILD; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteAggGroupKeyCondForPushDown(SOptimizeContext* pCxt, SAggLogicNode* pAgg, SNode* pGroupKeyCond) { + SRewriteAggGroupKeyCondContext cxt = {.pAgg = pAgg, .errCode = TSDB_CODE_SUCCESS}; + nodesRewriteExpr(&pGroupKeyCond, rewriteAggGroupKeyCondForPushDownImpl, &cxt); + return cxt.errCode; +} + +static int32_t pushDownCondOptDealAgg(SOptimizeContext* pCxt, SAggLogicNode* pAgg) { + if (NULL == pAgg->node.pConditions || + OPTIMIZE_FLAG_TEST_MASK(pAgg->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE)) { + return TSDB_CODE_SUCCESS; + } + //TODO: remove it after full implementation of pushing down to child + if (1 != LIST_LENGTH(pAgg->node.pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pAgg->node.pChildren, 0))) { + return TSDB_CODE_SUCCESS; + } + + SNode* pAggFuncCond = NULL; + SNode* pGroupKeyCond = NULL; + int32_t code = partitionAggCond(pAgg, &pAggFuncCond, &pGroupKeyCond); + if (TSDB_CODE_SUCCESS == code && NULL != pAggFuncCond) { + code = pushCondToAggCond(pCxt, pAgg, &pAggFuncCond); + } + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeyCond) { + code = rewriteAggGroupKeyCondForPushDown(pCxt, pAgg, pGroupKeyCond); + } + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeyCond) { + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0); + code = pushDownCondOptPushCondToChild(pCxt, pChild, &pGroupKeyCond); + } + if (TSDB_CODE_SUCCESS == code) { + OPTIMIZE_FLAG_SET_MASK(pAgg->node.optimizedFlag, OPTIMIZE_FLAG_PUSH_DOWN_CONDE); + pCxt->optimized = true; + } else { + nodesDestroyNode(pGroupKeyCond); + nodesDestroyNode(pAggFuncCond); + } + return code; +} + static int32_t pushDownCondOptimizeImpl(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pLogicNode)) { @@ -1668,8 +1817,8 @@ static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSelfNode) { SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSelfNode->pChildren, 0); - SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS}; + SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS}; nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt); int32_t code = cxt.errCode; From 3a159de7e2cead0584f852dded29bee9ee674f54 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 30 Jun 2022 15:28:03 +0800 Subject: [PATCH 03/24] refactor: assign the committed version --- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index dbbe3d3750..02a0783927 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -266,6 +266,8 @@ int vnodeCommit(SVnode *pVnode) { ASSERT(0); return -1; } + + pVnode->state.committed = info.state.committed; // apply the commit (TODO) vnodeBufPoolReset(pVnode->onCommit); From 4b73cdb529fc68260475cbdf4a6395ab3b8c4c5d Mon Sep 17 00:00:00 2001 From: cpwu Date: Thu, 30 Jun 2022 15:34:38 +0800 Subject: [PATCH 04/24] fix case --- tests/pytest/util/common.py | 53 ++++++++++++++--------- tests/pytest/util/constant.py | 28 ++++++++++++ tests/pytest/util/sql.py | 42 +++++++++++++++++- tests/system-test/2-query/explain.py | 32 -------------- tests/system-test/2-query/hyperloglog.py | 37 ++-------------- tests/system-test/2-query/leastsquares.py | 32 -------------- tests/system-test/2-query/spread.py | 32 -------------- 7 files changed, 105 insertions(+), 151 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 6caf5624a4..998e0248ab 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -17,6 +17,7 @@ import string import requests import time import socket +import json from .boundary import DataBoundary import taos from util.log import * @@ -25,7 +26,7 @@ from util.cases import * from util.dnodes import * from util.common import * -class TDCom: +class TDCom: def __init__(self): self.sml_type = None self.env_setting = None @@ -206,12 +207,12 @@ class TDCom: """ generate long name mode could be numbers/letters/letters_mixed/mixed - """ - if mode == "numbers": + """ + if mode == "numbers": chars = ''.join(random.choice(string.digits) for i in range(len)) - elif mode == "letters": + elif mode == "letters": chars = ''.join(random.choice(string.ascii_letters.lower()) for i in range(len)) - elif mode == "letters_mixed": + elif mode == "letters_mixed": chars = ''.join(random.choice(string.ascii_letters.upper() + string.ascii_letters.lower()) for i in range(len)) else: chars = ''.join(random.choice(string.ascii_letters.lower() + string.digits) for i in range(len)) @@ -276,7 +277,7 @@ class TDCom: vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions ''' sqlString = f'create database if not exists {dbName} ' - + dbParams = "" if len(kwargs) > 0: for param, value in kwargs.items(): @@ -306,7 +307,7 @@ class TDCom: # return # def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict): - # tsql.execute("use %s" %dbName) + # tsql.execute("use %s" %dbName) # tagsValues = '' # for i in range(tagDict['int']): # if i > 0: @@ -323,7 +324,7 @@ class TDCom: # sql = pre_create # if sql != pre_create: # tsql.execute(sql) - + # tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) # return @@ -352,7 +353,7 @@ class TDCom: # tsql.execute(sql) # tdLog.debug("insert data ............ [OK]") # return - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -367,7 +368,7 @@ class TDCom: if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break - return buildPath + return buildPath def getClientCfgPath(self): buildPath = self.getBuildPath() @@ -398,7 +399,7 @@ class TDCom: return newTdSql ################################################################################################################ - # port from the common.py of new test frame + # port from the common.py of new test frame ################################################################################################################ def gen_default_tag_str(self): default_tag_str = "" @@ -527,9 +528,9 @@ class TDCom: tag_value_list.append(self.gen_random_type_value(tag_elm["type"], "", "", "", "")) else: continue - return tag_value_list + return tag_value_list - def gen_column_value_list(self, column_elm_list, ts_value=None): + def gen_column_value_list(self, column_elm_list, ts_value=None): if ts_value is None: ts_value = self.genTs()[0] @@ -554,7 +555,7 @@ class TDCom: return column_value_list def create_stable(self, tsql, dbname=None, stbname="stb", column_elm_list=None, tag_elm_list=None, - count=1, default_stbname_prefix="stb", **kwargs): + count=1, default_stbname_prefix="stb", **kwargs): colname_prefix = 'c' tagname_prefix = 't' stbname_index_start_num = 1 @@ -589,20 +590,20 @@ class TDCom: tag_value_str += f'"{tag_value}", ' else: tag_value_str += f'{tag_value}, ' - tag_value_str = tag_value_str.rstrip()[:-1] - + tag_value_str = tag_value_str.rstrip()[:-1] + if int(count) <= 1: create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};' tsql.execute(create_ctable_sql) else: for _ in range(count): create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};' - ctbname_index_start_num += 1 + ctbname_index_start_num += 1 tdLog.info("create ctb sql: %s"%create_ctable_sql) tsql.execute(create_ctable_sql) - + def create_table(self, tsql, dbname=None, tbname="ntb", column_elm_list=None, count=1, **kwargs): - tbname_index_start_num = 1 + tbname_index_start_num = 1 tbname_prefix="ntb" tb_params = "" @@ -632,7 +633,7 @@ class TDCom: column_value_str += f'"{column_value}", ' else: column_value_str += f'{column_value}, ' - column_value_str = column_value_str.rstrip()[:-1] + column_value_str = column_value_str.rstrip()[:-1] if int(count) <= 1: insert_sql = f'insert into {self.tb_name} values ({column_value_str});' tsql.execute(insert_sql) @@ -651,4 +652,16 @@ class TDCom: return res_list else: tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}") + + +def is_json(msg): + if isinstance(msg, str): + try: + json.loads(msg) + return True + except: + return False + else: + return False + tdCom = TDCom() diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py index 5be1c603b3..83487da023 100644 --- a/tests/pytest/util/constant.py +++ b/tests/pytest/util/constant.py @@ -71,6 +71,34 @@ TAOS_KEYWORDS = [ "COPY", "IF", "NOW", "STABLES", "WHERE", ] +NUM_FUNC = [ + "ABS", "ACOS", "ASIN", "ATAN", "CEIL", "COS", "FLOOR", "LOG", "POW", "ROUND", "SIN", "SQRT", "TAN", +] + +STR_FUNC = [ + "CHAR_LENGTH", "CONCAT", "CONCAT_WS", "LENGTH", "LOWER","LTRIM", "RTRIM", "SUBSTR", "UPPER", +] + +CONVER_FUNC = ["CASR", "TO_ISO8601", "TO_JSON", "TP_UNIXTIMESTAMP"] + +SELECT_FUNC = [ + "APERCENTILE", "BOTTOM", "FIRST", "INTERP", "LAST", "MAX", "MIN", "PERCENTILE", "TAIL", "TOP", "UNIQUE", +] + +AGG_FUNC = [ + "AVG", "COUNT", "ELAPSED", "LEASTSQUARES", "MODE", "SPREAD", "STDDEV", "SUM", "HYPERLOGLOG", "HISTOGRAM", +] + +TS_FUNC = [ + "CSUM", "DERIVATIVE", "DIFF", "IRATE", "MAVG", "SAMPLE", "STATECOUNT", "STATEDURATION", "TWA" +] + +SYSINFO_FUCN = [ + "DATABASE", "CLIENT_VERSION", "SERVER_VERSION", "SERVER_STATUS", "CURRENT_USER", "USER" +] + + + # basic data type boundary TINYINT_MAX = 127 TINYINT_MIN = -128 diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index fe802dd9a3..585594e035 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -96,6 +96,15 @@ class TDSql: return self.queryResult return self.queryRows + def is_err_sql(self, sql): + err_flag = True + try: + self.cursor.execute(sql) + except BaseException: + err_flag = False + + return False if err_flag else True + def getVariable(self, search_attr): ''' get variable of search_attr access "show variables" @@ -249,7 +258,6 @@ class TDSql: raise Exception(repr(e)) return self.queryResult - def executeTimes(self, sql, times): for i in range(times): try: @@ -336,6 +344,38 @@ class TDSql: elif precision == "ns": return int(times*1000*1000) + def get_type(self, col): + if self.cursor.istype(col, "BOOL"): + return "BOOL" + if self.cursor.istype(col, "INT"): + return "INT" + if self.cursor.istype(col, "BIGINT"): + return "BIGINT" + if self.cursor.istype(col, "TINYINT"): + return "TINYINT" + if self.cursor.istype(col, "SMALLINT"): + return "SMALLINT" + if self.cursor.istype(col, "FLOAT"): + return "FLOAT" + if self.cursor.istype(col, "DOUBLE"): + return "DOUBLE" + if self.cursor.istype(col, "BINARY"): + return "BINARY" + if self.cursor.istype(col, "NCHAR"): + return "NCHAR" + if self.cursor.istype(col, "TIMESTAMP"): + return "TIMESTAMP" + if self.cursor.istype(col, "JSON"): + return "JSON" + if self.cursor.istype(col, "TINYINT UNSIGNED"): + return "TINYINT UNSIGNED" + if self.cursor.istype(col, "SMALLINT UNSIGNED"): + return "SMALLINT UNSIGNED" + if self.cursor.istype(col, "INT UNSIGNED"): + return "INT UNSIGNED" + if self.cursor.istype(col, "BIGINT UNSIGNED"): + return "BIGINT UNSIGNED" + def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 diff --git a/tests/system-test/2-query/explain.py b/tests/system-test/2-query/explain.py index d440144841..f2c6cb7771 100644 --- a/tests/system-test/2-query/explain.py +++ b/tests/system-test/2-query/explain.py @@ -163,38 +163,6 @@ class TDTestCase: # return filter(None, sqls) return list(filter(None, sqls)) - def __get_type(self, col): - if tdSql.cursor.istype(col, "BOOL"): - return "BOOL" - if tdSql.cursor.istype(col, "INT"): - return "INT" - if tdSql.cursor.istype(col, "BIGINT"): - return "BIGINT" - if tdSql.cursor.istype(col, "TINYINT"): - return "TINYINT" - if tdSql.cursor.istype(col, "SMALLINT"): - return "SMALLINT" - if tdSql.cursor.istype(col, "FLOAT"): - return "FLOAT" - if tdSql.cursor.istype(col, "DOUBLE"): - return "DOUBLE" - if tdSql.cursor.istype(col, "BINARY"): - return "BINARY" - if tdSql.cursor.istype(col, "NCHAR"): - return "NCHAR" - if tdSql.cursor.istype(col, "TIMESTAMP"): - return "TIMESTAMP" - if tdSql.cursor.istype(col, "JSON"): - return "JSON" - if tdSql.cursor.istype(col, "TINYINT UNSIGNED"): - return "TINYINT UNSIGNED" - if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"): - return "SMALLINT UNSIGNED" - if tdSql.cursor.istype(col, "INT UNSIGNED"): - return "INT UNSIGNED" - if tdSql.cursor.istype(col, "BIGINT UNSIGNED"): - return "BIGINT UNSIGNED" - def explain_check(self): sqls = self.sql_list() tdLog.printNoPrefix("===step 1: curent case, must return query OK") diff --git a/tests/system-test/2-query/hyperloglog.py b/tests/system-test/2-query/hyperloglog.py index 8dd6bd2dda..6ab971173a 100644 --- a/tests/system-test/2-query/hyperloglog.py +++ b/tests/system-test/2-query/hyperloglog.py @@ -116,37 +116,6 @@ class TDTestCase: # return filter(None, sqls) return list(filter(None, sqls)) - def __get_type(self, col): - if tdSql.cursor.istype(col, "BOOL"): - return "BOOL" - if tdSql.cursor.istype(col, "INT"): - return "INT" - if tdSql.cursor.istype(col, "BIGINT"): - return "BIGINT" - if tdSql.cursor.istype(col, "TINYINT"): - return "TINYINT" - if tdSql.cursor.istype(col, "SMALLINT"): - return "SMALLINT" - if tdSql.cursor.istype(col, "FLOAT"): - return "FLOAT" - if tdSql.cursor.istype(col, "DOUBLE"): - return "DOUBLE" - if tdSql.cursor.istype(col, "BINARY"): - return "BINARY" - if tdSql.cursor.istype(col, "NCHAR"): - return "NCHAR" - if tdSql.cursor.istype(col, "TIMESTAMP"): - return "TIMESTAMP" - if tdSql.cursor.istype(col, "JSON"): - return "JSON" - if tdSql.cursor.istype(col, "TINYINT UNSIGNED"): - return "TINYINT UNSIGNED" - if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"): - return "SMALLINT UNSIGNED" - if tdSql.cursor.istype(col, "INT UNSIGNED"): - return "INT UNSIGNED" - if tdSql.cursor.istype(col, "BIGINT UNSIGNED"): - return "BIGINT UNSIGNED" def hyperloglog_check(self): sqls = self.sql_list() @@ -228,10 +197,10 @@ class TDTestCase: tag_sql += f"{k} {v}," tdSql.execute(f'create table if not exists {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})') - def __insert_data(self): + def __insert_data(self): pass - + def __hyperloglog_check_distribute(self): dbname = "dbtest" stbname = "stb" @@ -286,7 +255,7 @@ class TDTestCase: tdSql.query(f"select hyperloglog({k}) from {stbname} group by {k}") tdSql.execute(f'drop database {dbname}') - + def __insert_data(self, rows): now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) diff --git a/tests/system-test/2-query/leastsquares.py b/tests/system-test/2-query/leastsquares.py index e8fa32e8b3..3bab509942 100644 --- a/tests/system-test/2-query/leastsquares.py +++ b/tests/system-test/2-query/leastsquares.py @@ -195,38 +195,6 @@ class TDTestCase: # return filter(None, sqls) return list(filter(None, current_sqls)), list(filter(None, err_sqls)) - def __get_type(self, col): - if tdSql.cursor.istype(col, "BOOL"): - return "BOOL" - if tdSql.cursor.istype(col, "INT"): - return "INT" - if tdSql.cursor.istype(col, "BIGINT"): - return "BIGINT" - if tdSql.cursor.istype(col, "TINYINT"): - return "TINYINT" - if tdSql.cursor.istype(col, "SMALLINT"): - return "SMALLINT" - if tdSql.cursor.istype(col, "FLOAT"): - return "FLOAT" - if tdSql.cursor.istype(col, "DOUBLE"): - return "DOUBLE" - if tdSql.cursor.istype(col, "BINARY"): - return "BINARY" - if tdSql.cursor.istype(col, "NCHAR"): - return "NCHAR" - if tdSql.cursor.istype(col, "TIMESTAMP"): - return "TIMESTAMP" - if tdSql.cursor.istype(col, "JSON"): - return "JSON" - if tdSql.cursor.istype(col, "TINYINT UNSIGNED"): - return "TINYINT UNSIGNED" - if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"): - return "SMALLINT UNSIGNED" - if tdSql.cursor.istype(col, "INT UNSIGNED"): - return "INT UNSIGNED" - if tdSql.cursor.istype(col, "BIGINT UNSIGNED"): - return "BIGINT UNSIGNED" - def leastsquares_check(self): current_sqls, err_sqls = self.sql_list() for i in range(len(err_sqls)): diff --git a/tests/system-test/2-query/spread.py b/tests/system-test/2-query/spread.py index d2dbbd03ed..51c569e565 100644 --- a/tests/system-test/2-query/spread.py +++ b/tests/system-test/2-query/spread.py @@ -159,38 +159,6 @@ class TDTestCase: # return filter(None, sqls) return list(filter(None, sqls)) - def __get_type(self, col): - if tdSql.cursor.istype(col, "BOOL"): - return "BOOL" - if tdSql.cursor.istype(col, "INT"): - return "INT" - if tdSql.cursor.istype(col, "BIGINT"): - return "BIGINT" - if tdSql.cursor.istype(col, "TINYINT"): - return "TINYINT" - if tdSql.cursor.istype(col, "SMALLINT"): - return "SMALLINT" - if tdSql.cursor.istype(col, "FLOAT"): - return "FLOAT" - if tdSql.cursor.istype(col, "DOUBLE"): - return "DOUBLE" - if tdSql.cursor.istype(col, "BINARY"): - return "BINARY" - if tdSql.cursor.istype(col, "NCHAR"): - return "NCHAR" - if tdSql.cursor.istype(col, "TIMESTAMP"): - return "TIMESTAMP" - if tdSql.cursor.istype(col, "JSON"): - return "JSON" - if tdSql.cursor.istype(col, "TINYINT UNSIGNED"): - return "TINYINT UNSIGNED" - if tdSql.cursor.istype(col, "SMALLINT UNSIGNED"): - return "SMALLINT UNSIGNED" - if tdSql.cursor.istype(col, "INT UNSIGNED"): - return "INT UNSIGNED" - if tdSql.cursor.istype(col, "BIGINT UNSIGNED"): - return "BIGINT UNSIGNED" - def spread_check(self): sqls = self.sql_list() tdLog.printNoPrefix("===step 1: curent case, must return query OK") From 7ef4d3f03a1f6a034a66812b5cab9db0e44db5d4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Jun 2022 15:39:30 +0800 Subject: [PATCH 05/24] fix: fix null as tag value issue --- source/libs/parser/src/parTranslater.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 521c733ceb..6dc14dbf15 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -783,6 +783,12 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD if (pVal->placeholderNo > 0 || pVal->isNull) { return DEAL_RES_CONTINUE; } + if (TSDB_DATA_TYPE_NULL == pVal->node.resType.type) { + // TODO + //pVal->node.resType = targetDt; + pVal->isNull = true; + return DEAL_RES_CONTINUE; + } if (pVal->isDuration) { if (parseNatualDuration(pVal->literal, strlen(pVal->literal), &pVal->datum.i, &pVal->unit, precision) != TSDB_CODE_SUCCESS) { @@ -5335,7 +5341,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau if (code != TSDB_CODE_SUCCESS) { goto end; } - } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL && !pVal->isNull) { char* tmpVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { From 7c3f43d6d59c0b181f980a2568d7fd6101b8f3d2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 30 Jun 2022 16:20:43 +0800 Subject: [PATCH 06/24] refactor: tsma logic optimization --- source/dnode/vnode/src/sma/smaTimeRange.c | 63 ++++++++++++++--------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 2244b91c28..ecd8db8ea9 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -113,7 +113,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { if (!pDataBlocks) { terrno = TSDB_CODE_TSMA_INVALID_PTR; smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); - return terrno; + return TSDB_CODE_FAILED; } if (taosArrayGetSize(pDataBlocks) <= 0) { @@ -127,9 +127,9 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { return TSDB_CODE_FAILED; } - SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); - SSmaStat *pStat = NULL; - STSmaStat *pItem = NULL; + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = NULL; + STSmaStat *pTsmaStat = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { terrno = TSDB_CODE_TSMA_INVALID_STAT; @@ -137,32 +137,43 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } tdRefSmaStat(pSma, pStat); - pItem = &pStat->tsmaStat; + pTsmaStat = SMA_TSMA_STAT(pStat); - ASSERT(pItem); - - if (!pItem->pTSma) { + if (!pTsmaStat->pTSma) { STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); if (!pTSma) { - terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META; - smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); - return TSDB_CODE_FAILED; + smaError("vgId:%d, failed to get STSma while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + indexUid, tstrerror(terrno)); + goto _err; + } + pTsmaStat->pTSma = pTSma; + pTsmaStat->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1); + if (!pTsmaStat->pTSchema) { + smaError("vgId:%d, failed to get STSchema while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + indexUid, tstrerror(terrno)); + goto _err; } - pItem->pTSma = pTSma; - pItem->pTSchema = metaGetTbTSchema(SMA_META(pSma), pTSma->dstTbUid, -1); - ASSERT(pItem->pTSchema); // TODO } - ASSERT(pItem->pTSma->indexUid == indexUid); + if (pTsmaStat->pTSma->indexUid != indexUid) { + terrno = TSDB_CODE_VND_APP_ERROR; + smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 "(!=%" PRIi64 ") failed since %s", SMA_VID(pSma), indexUid, + pTsmaStat->pTSma->indexUid, tstrerror(terrno)); + goto _err; + } - SSubmitReq *pSubmitReq = NULL; + SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, + pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId); - pSubmitReq = tdBlockToSubmit((const SArray *)msg, pItem->pTSchema, true, pItem->pTSma->dstTbUid, - pItem->pTSma->dstTbName, pItem->pTSma->dstVgId); + if (!pSubmitReq) { + smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + indexUid, tstrerror(terrno)); + goto _err; + } - ASSERT(pSubmitReq); // TODO - - ASSERT(!strncasecmp("td.tsma.rst.tb", pItem->pTSma->dstTbName, 14)); +#if 0 + ASSERT(!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)); +#endif SRpcMsg submitReqMsg = { .msgType = TDMT_VND_SUBMIT, @@ -170,9 +181,15 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { .contLen = ntohl(pSubmitReq->length), }; - ASSERT(tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) == 0); + if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { + smaError("vgId:%d, failed to put SubmitReq msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + indexUid, tstrerror(terrno)); + goto _err; + } tdUnRefSmaStat(pSma, pStat); - return TSDB_CODE_SUCCESS; +_err: + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; } \ No newline at end of file From 860bf88f3124bcc63686a363203202cd420de48d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 17:20:04 +0800 Subject: [PATCH 07/24] feat(query): add last_row function without cache TD-16927 --- include/libs/function/functionMgt.h | 1 + source/libs/function/inc/builtinsimpl.h | 3 + source/libs/function/src/builtins.c | 11 +-- source/libs/function/src/builtinsimpl.c | 126 ++++++++++++++++++++---- 4 files changed, 116 insertions(+), 25 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f03422672d..582cddaf0f 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -34,6 +34,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_ELAPSED, FUNCTION_TYPE_IRATE, FUNCTION_TYPE_LAST_ROW, + FUNCTION_TYPE_LAST_ROWT, FUNCTION_TYPE_MAX, FUNCTION_TYPE_MIN, FUNCTION_TYPE_MODE, diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f7e22cb151..8eafb0703e 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -117,6 +117,9 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getFirstLastInfoSize(int32_t resBytes); +int32_t lastRowFunction(SqlFunctionCtx *pCtx); +int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool topBotFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23d7185508..9ea07db41e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1947,17 +1947,14 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = irateFinalize }, { - .name = "last_row", - .type = FUNCTION_TYPE_LAST_ROW, + .name = "last_rowt", + .type = FUNCTION_TYPE_LAST_ROWT, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = lastFunction, - .finalizeFunc = firstLastFinalize, - .pPartialFunc = "_last_partial", - .pMergeFunc = "_last_merge", - .combineFunc = lastCombine, + .processFunc = lastRowFunction, + .finalizeFunc = lastRowFinalize, }, { .name = "_cache_last_row", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a75d5a2ecb..cb0c991389 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -80,6 +80,7 @@ typedef struct STopBotRes { typedef struct SFirstLastRes { bool hasResult; + bool isNull; //used for last_row function only int32_t bytes; char buf[]; } SFirstLastRes; @@ -2597,15 +2598,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; - //handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } pInfo->hasResult = true; //DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; @@ -2629,15 +2621,6 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; - //handle selectivity - if (pCtx->subsidiaries.num > 0) { - STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); - if (!pInfo->hasResult) { - saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } else { - copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); - } - } pInfo->hasResult = true; pResInfo->numOfRes = 1; //DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -2763,6 +2746,113 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return TSDB_CODE_SUCCESS; } +int32_t lastRowFunction(SqlFunctionCtx* pCtx) { + int32_t numOfElems = 0; + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + int32_t type = pInputCol->info.type; + int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; + + SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL; + + TSKEY startKey = getRowPTs(pInput->pPTS, 0); + TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); + + int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + + if (blockDataOrder == TSDB_ORDER_ASC) { + for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + pInfo->isNull = true; + } else { + pInfo->isNull = false; + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataTLen(data); + pInfo->bytes = bytes; + } + memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); + } + *(TSKEY*)(pInfo->buf) = cts; + numOfElems++; + //handle selectivity + if (pCtx->subsidiaries.num > 0) { + STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); + if (!pInfo->hasResult) { + saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } else { + copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } + } + pInfo->hasResult = true; + //DO_UPDATE_TAG_COLUMNS(pCtx, ts); + pResInfo->numOfRes = 1; + } + break; + } + } else { // descending order + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + pInfo->isNull = true; + } else { + pInfo->isNull = false; + if (IS_VAR_DATA_TYPE(type)) { + bytes = varDataTLen(data); + pInfo->bytes = bytes; + } + memcpy(pInfo->buf + sizeof(TSKEY), data, bytes); + } + *(TSKEY*)(pInfo->buf) = cts; + numOfElems++; + //handle selectivity + if (pCtx->subsidiaries.num > 0) { + STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); + if (!pInfo->hasResult) { + saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } else { + copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } + } + pInfo->hasResult = true; + pResInfo->numOfRes = 1; + //DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + break; + } + } + + SET_VAL(pResInfo, numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + + +int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, pRes->buf + sizeof(TSKEY), pRes->isNull); + //handle selectivity + STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY)); + setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows); + + return pResInfo->numOfRes; +} + + bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; From 82bd8a560f0f522f96de5065ee0fc6f6bcef7d9f Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Jun 2022 17:28:49 +0800 Subject: [PATCH 08/24] fix: reserved column should not be copied --- source/libs/executor/src/executorimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 95782ce001..e2e281d877 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1348,7 +1348,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. - if (pDst->pData == NULL) { + if (pDst->pData == NULL || pSrc->pData == NULL) { continue; } From 1dd89be98bc42b799061678109f3926b75d7fab4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 17:39:20 +0800 Subject: [PATCH 09/24] enh(query): add last_row function without cache TD-16927 --- include/libs/function/functionMgt.h | 2 +- source/libs/function/src/builtins.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 582cddaf0f..5474315434 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -34,7 +34,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_ELAPSED, FUNCTION_TYPE_IRATE, FUNCTION_TYPE_LAST_ROW, - FUNCTION_TYPE_LAST_ROWT, + FUNCTION_TYPE_LAST_ROWT, //TODO: removed FUNCTION_TYPE_MAX, FUNCTION_TYPE_MIN, FUNCTION_TYPE_MODE, diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index bc3b0f165e..b3a627271f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1952,7 +1952,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = irateFinalize }, { - .name = "last_rowt", + .name = "last_row", .type = FUNCTION_TYPE_LAST_ROWT, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, From e38e23a3ecb6e3a9913d1920ca0fa4ab43623e38 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 17:45:55 +0800 Subject: [PATCH 10/24] fix error --- source/libs/function/src/builtinsimpl.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 6bff978e69..eb21ee1fb5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2598,6 +2598,15 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; + //handle selectivity + if (pCtx->subsidiaries.num > 0) { + STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); + if (!pInfo->hasResult) { + saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } else { + copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } + } pInfo->hasResult = true; //DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; @@ -2621,6 +2630,15 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; + //handle selectivity + if (pCtx->subsidiaries.num > 0) { + STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY)); + if (!pInfo->hasResult) { + saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } else { + copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos); + } + } pInfo->hasResult = true; pResInfo->numOfRes = 1; //DO_UPDATE_TAG_COLUMNS(pCtx, ts); From 0692ab4c5e8ec99806af766a45746134b83dcdec Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 17:55:54 +0800 Subject: [PATCH 11/24] enh(query): add last_row selectivity --- source/libs/function/src/builtins.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b3a627271f..68d89e9ec8 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1954,7 +1954,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROWT, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, From cda2f844ced4d498cdc249c54f1a0f4666580ac7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Jun 2022 18:12:04 +0800 Subject: [PATCH 12/24] fix: fix json tag null issue --- source/libs/parser/src/parTranslater.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6dc14dbf15..762a038bb1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -786,6 +786,7 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD if (TSDB_DATA_TYPE_NULL == pVal->node.resType.type) { // TODO //pVal->node.resType = targetDt; + pVal->translate = true; pVal->isNull = true; return DEAL_RES_CONTINUE; } @@ -5608,8 +5609,8 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } - if (DEAL_RES_ERROR == - translateValueImpl(pCxt, pStmt->pVal, schemaToDataType(pTableMeta->tableInfo.precision, pSchema))) { + SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); + if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt)) { return pCxt->errCode; } @@ -5618,7 +5619,8 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS } pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); - if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) { + if (targetDt.type == TSDB_DATA_TYPE_JSON) { + pReq->isNull = 0; if (pStmt->pVal->literal && strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal); From afe67a609ac616041271feb82366178e45aa97fa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 30 Jun 2022 19:16:22 +0800 Subject: [PATCH 13/24] add query epset --- source/libs/transport/src/transCli.c | 5 +++-- source/libs/transport/src/transSvr.c | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2836bc2949..1620760b81 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -607,9 +607,10 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { uv_close((uv_handle_t*)conn->stream, cliDestroy); - } else { - cliDestroy((uv_handle_t*)conn->stream); } + //} else { + // cliDestroy((uv_handle_t*)conn->stream); + //} } } static void cliDestroy(uv_handle_t* handle) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index dfc6de6442..6a964b4baf 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -140,7 +140,6 @@ static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd); static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, uvHandleRegister, NULL}; - static void uvDestroyConn(uv_handle_t* handle); // server and worker thread @@ -777,9 +776,10 @@ static void destroyConn(SSvrConn* conn, bool clear) { if (!uv_is_closing((uv_handle_t*)conn->pTcp)) { tTrace("conn %p to be destroyed", conn); uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); - } else { - uvDestroyConn((uv_handle_t*)conn->pTcp); } + //} else { + // uvDestroyConn((uv_handle_t*)conn->pTcp); + //} } } static void destroyConnRegArg(SSvrConn* conn) { From 61619c940f09d80cb545696007d6de413309ea65 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 19:43:48 +0800 Subject: [PATCH 14/24] fix some test cases --- tests/system-test/0-others/udfTest.py | 92 +++++++++--------- tests/system-test/0-others/udf_create.py | 92 +++++++++--------- .../system-test/0-others/udf_restart_taosd.py | 96 ++++++++++--------- 3 files changed, 146 insertions(+), 134 deletions(-) diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 375b7a6272..40f803432a 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -32,7 +32,7 @@ class TDTestCase: buildPath = root[:len(root) - len("/build/bin")] break return buildPath - + def prepare_udf_so(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -58,7 +58,7 @@ class TDTestCase: def prepare_data(self): - + tdSql.execute("drop database if exists db ") tdSql.execute("create database if not exists db duration 300") tdSql.execute("use db") @@ -68,7 +68,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -150,7 +150,7 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) - + functions = tdSql.getResult("show functions") function_nums = len(functions) if function_nums == 2: @@ -175,14 +175,14 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) - + functions = tdSql.getResult("show functions") function_nums = len(functions) if function_nums == 2: tdLog.info("create two udf functions success ") - + def basic_udf_query(self): - + # scalar functions tdSql.execute("use db ") @@ -256,7 +256,7 @@ class TDTestCase: tdSql.checkData(0,1,165.247614504) tdSql.checkData(0,2,2551.470164435) tdSql.checkData(0,3,2.652476145) - + # # bug for crash when query sub table tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") tdSql.checkData(0,0,378.215547010) @@ -281,7 +281,7 @@ class TDTestCase: tdSql.error("select udf1(num1) , stddev(num1) from tb;") tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") - # stable + # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") tdSql.error("select udf1(c1) , avg(c1) from stb1;") tdSql.error("select udf1(c1) , twa(c1) from stb1;") @@ -302,23 +302,25 @@ class TDTestCase: tdSql.query("select ceil(num1) , min(num1) from tb;") tdSql.checkRows(1) tdSql.query("select udf1(num1) , first(num1) from tb;") - + tdSql.query("select abs(num1) , first(num1) from tb;") - + tdSql.query("select udf1(num1) , last(num1) from tb;") - + tdSql.query("select round(num1) , last(num1) from tb;") - + tdSql.query("select udf1(num1) , top(num1,1) from tb;") tdSql.checkRows(1) tdSql.query("select udf1(num1) , bottom(num1,1) from tb;") tdSql.checkRows(1) - tdSql.error("select udf1(num1) , last_row(num1) from tb;") - - tdSql.error("select round(num1) , last_row(num1) from tb;") - - - # stable + tdSql.query("select udf1(num1) , last_row(num1) from tb;") + tdSql.checkRows(1) + + tdSql.query("select round(num1) , last_row(num1) from tb;") + tdSql.checkRows(1) + + + # stable tdSql.query("select udf1(c1) , max(c1) from stb1;") tdSql.checkRows(1) tdSql.query("select abs(c1) , max(c1) from stb1;") @@ -328,9 +330,9 @@ class TDTestCase: tdSql.query("select floor(c1) , min(c1) from stb1;") tdSql.checkRows(1) tdSql.query("select udf1(c1) , first(c1) from stb1;") - + tdSql.query("select udf1(c1) , last(c1) from stb1;") - + tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;") tdSql.checkRows(1) tdSql.query("select abs(c1) , top(c1 ,1) from stb1;") @@ -340,9 +342,11 @@ class TDTestCase: tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;") tdSql.checkRows(1) - tdSql.error("select udf1(c1) , last_row(c1) from stb1;") - tdSql.error("select ceil(c1) , last_row(c1) from stb1;") - + tdSql.query("select udf1(c1) , last_row(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select ceil(c1) , last_row(c1) from stb1;") + tdSql.checkRows(1) + # regular table with compute functions tdSql.query("select udf1(num1) , abs(num1) from tb;") @@ -350,7 +354,7 @@ class TDTestCase: tdSql.query("select floor(num1) , abs(num1) from tb;") tdSql.checkRows(12) - # # bug need fix + # # bug need fix #tdSql.query("select udf1(num1) , csum(num1) from tb;") #tdSql.checkRows(9) @@ -382,8 +386,8 @@ class TDTestCase: tdSql.checkData(1,0,88) tdSql.checkData(1,1,7) - # bug fix for crash - # order by udf function result + # bug fix for crash + # order by udf function result for _ in range(50): tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") print(tdSql.queryResult) @@ -401,7 +405,7 @@ class TDTestCase: tdSql.checkData(0,1,88) tdSql.checkData(0,2,-99.990000000) tdSql.checkData(0,3,88) - + tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) tdSql.checkData(0,1,0) @@ -429,7 +433,7 @@ class TDTestCase: tdSql.checkData(0,1,168.819430161) tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - # udf functions with group by + # udf functions with group by tdSql.query("select udf1(c1) from ct1 group by c1") tdSql.checkRows(10) tdSql.query("select udf1(c1) from stb1 group by c1") @@ -452,7 +456,7 @@ class TDTestCase: tdSql.query("select udf2(c1) from stb1 group by floor(c1)") tdSql.checkRows(11) - # udf mix with order by + # udf mix with order by tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") tdSql.checkRows(11) @@ -481,7 +485,7 @@ class TDTestCase: tdSql.checkData(0,1,169.661427555) def try_query_sql(self): - udf1_sqls = [ + udf1_sqls = [ "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" , "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" , "select udf1(num1) , max(num1) from tb;" , @@ -525,7 +529,7 @@ class TDTestCase: "select udf2(c1) from stb1 group by udf1(c1)" , "select udf2(c1) from stb1 group by floor(c1)" , "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" , - + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , @@ -551,7 +555,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - # create function without aggregate + # create function without aggregate tdLog.info(" create function with out aggregate ") tdSql.query("drop function udf1 ") @@ -575,8 +579,8 @@ class TDTestCase: tdSql.error(" select test(c1) from stb1 ") tdSql.error(" select test(c1,c6), test(c6) from stb1 ") tdSql.error(" select test(num1,num2), test(num1) from tb ") - - + + def loop_kill_udfd(self): @@ -585,7 +589,7 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - + cfgPath = buildPath + "/../sim/dnode1/cfg" udfdPath = buildPath +'/build/bin/udfd' @@ -596,19 +600,19 @@ class TDTestCase: tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # stop udfd cmds + # stop udfd cmds get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'" processID = subprocess.check_output(get_processID, shell=True).decode("utf-8") stop_udfd = " kill -9 %s" % processID os.system(stop_udfd) time.sleep(2) - + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # # start udfd cmds + # # start udfd cmds # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &" # tdLog.info("start udfd : %s " % start_udfd) @@ -643,11 +647,11 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.start(1) time.sleep(2) - - + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - - print(" env is ok for all ") + + print(" env is ok for all ") self.prepare_udf_so() self.prepare_data() self.create_udf_function() @@ -659,7 +663,7 @@ class TDTestCase: time.sleep(2) self.basic_udf_query() self.test_function_name() - + def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index 5f3ab2e863..63650d6edc 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -34,7 +34,7 @@ class TDTestCase: buildPath = root[:len(root) - len("/build/bin")] break return buildPath - + def prepare_udf_so(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -60,7 +60,7 @@ class TDTestCase: def prepare_data(self): - + tdSql.execute("drop database if exists db ") tdSql.execute("create database if not exists db duration 300") tdSql.execute("use db") @@ -70,7 +70,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -152,7 +152,7 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) - + functions = tdSql.getResult("show functions") function_nums = len(functions) if function_nums == 2: @@ -177,14 +177,14 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) - + functions = tdSql.getResult("show functions") function_nums = len(functions) if function_nums == 2: tdLog.info("create two udf functions success ") - + def basic_udf_query(self): - + # scalar functions tdSql.execute("use db ") @@ -258,7 +258,7 @@ class TDTestCase: tdSql.checkData(0,1,165.247614504) tdSql.checkData(0,2,2551.470164435) tdSql.checkData(0,3,2.652476145) - + # # bug for crash when query sub table tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") tdSql.checkData(0,0,378.215547010) @@ -283,7 +283,7 @@ class TDTestCase: tdSql.error("select udf1(num1) , stddev(num1) from tb;") tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") - # stable + # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") tdSql.error("select udf1(c1) , avg(c1) from stb1;") tdSql.error("select udf1(c1) , twa(c1) from stb1;") @@ -304,23 +304,25 @@ class TDTestCase: tdSql.query("select ceil(num1) , min(num1) from tb;") tdSql.checkRows(1) tdSql.query("select udf1(num1) , first(num1) from tb;") - + tdSql.query("select abs(num1) , first(num1) from tb;") - + tdSql.query("select udf1(num1) , last(num1) from tb;") - + tdSql.query("select round(num1) , last(num1) from tb;") - + tdSql.query("select udf1(num1) , top(num1,1) from tb;") tdSql.checkRows(1) tdSql.query("select udf1(num1) , bottom(num1,1) from tb;") tdSql.checkRows(1) - tdSql.error("select udf1(num1) , last_row(num1) from tb;") - - tdSql.error("select round(num1) , last_row(num1) from tb;") - - - # stable + tdSql.query("select udf1(num1) , last_row(num1) from tb;") + tdSql.checkRows(1) + + tdSql.query("select round(num1) , last_row(num1) from tb;") + tdSql.checkRows(1) + + + # stable tdSql.query("select udf1(c1) , max(c1) from stb1;") tdSql.checkRows(1) tdSql.query("select abs(c1) , max(c1) from stb1;") @@ -330,9 +332,9 @@ class TDTestCase: tdSql.query("select floor(c1) , min(c1) from stb1;") tdSql.checkRows(1) tdSql.query("select udf1(c1) , first(c1) from stb1;") - + tdSql.query("select udf1(c1) , last(c1) from stb1;") - + tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;") tdSql.checkRows(1) tdSql.query("select abs(c1) , top(c1 ,1) from stb1;") @@ -342,9 +344,11 @@ class TDTestCase: tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;") tdSql.checkRows(1) - tdSql.error("select udf1(c1) , last_row(c1) from stb1;") - tdSql.error("select ceil(c1) , last_row(c1) from stb1;") - + tdSql.query("select udf1(c1) , last_row(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select ceil(c1) , last_row(c1) from stb1;") + tdSql.checkRows(1) + # regular table with compute functions tdSql.query("select udf1(num1) , abs(num1) from tb;") @@ -352,7 +356,7 @@ class TDTestCase: tdSql.query("select floor(num1) , abs(num1) from tb;") tdSql.checkRows(12) - # # bug need fix + # # bug need fix #tdSql.query("select udf1(num1) , csum(num1) from tb;") #tdSql.checkRows(9) @@ -384,8 +388,8 @@ class TDTestCase: tdSql.checkData(1,0,88) tdSql.checkData(1,1,7) - # bug fix for crash - # order by udf function result + # bug fix for crash + # order by udf function result for _ in range(50): tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") print(tdSql.queryResult) @@ -403,7 +407,7 @@ class TDTestCase: tdSql.checkData(0,1,88) tdSql.checkData(0,2,-99.990000000) tdSql.checkData(0,3,88) - + tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) tdSql.checkData(0,1,0) @@ -431,7 +435,7 @@ class TDTestCase: tdSql.checkData(0,1,168.819430161) tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - # udf functions with group by + # udf functions with group by tdSql.query("select udf1(c1) from ct1 group by c1") tdSql.checkRows(10) tdSql.query("select udf1(c1) from stb1 group by c1") @@ -454,7 +458,7 @@ class TDTestCase: tdSql.query("select udf2(c1) from stb1 group by floor(c1)") tdSql.checkRows(11) - # udf mix with order by + # udf mix with order by tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") tdSql.checkRows(11) @@ -483,7 +487,7 @@ class TDTestCase: tdSql.checkData(0,1,169.661427555) def try_query_sql(self): - udf1_sqls = [ + udf1_sqls = [ "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" , "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" , "select udf1(num1) , max(num1) from tb;" , @@ -527,7 +531,7 @@ class TDTestCase: "select udf2(c1) from stb1 group by udf1(c1)" , "select udf2(c1) from stb1 group by floor(c1)" , "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" , - + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , @@ -563,7 +567,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - # create function without aggregate + # create function without aggregate tdLog.info(" create function with out aggregate ") tdSql.query("drop function udf1 ") @@ -587,8 +591,8 @@ class TDTestCase: tdSql.error(" select test(c1) from stb1 ") tdSql.error(" select test(c1,c6), test(c6) from stb1 ") tdSql.error(" select test(num1,num2), test(num1) from tb ") - - + + def loop_kill_udfd(self): @@ -597,7 +601,7 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - + cfgPath = buildPath + "/../sim/dnode1/cfg" udfdPath = buildPath +'/build/bin/udfd' @@ -608,19 +612,19 @@ class TDTestCase: tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # stop udfd cmds + # stop udfd cmds get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'" processID = subprocess.check_output(get_processID, shell=True).decode("utf-8") stop_udfd = " kill -9 %s" % processID os.system(stop_udfd) time.sleep(2) - + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # # start udfd cmds + # # start udfd cmds # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &" # tdLog.info("start udfd : %s " % start_udfd) @@ -655,17 +659,17 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.start(1) time.sleep(2) - - + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - - print(" env is ok for all ") + + print(" env is ok for all ") self.prepare_udf_so() self.prepare_data() self.create_udf_function() self.basic_udf_query() self.unexpected_create() - + def stop(self): tdSql.close() diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index 857921e32c..c318980b67 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -31,7 +31,7 @@ class TDTestCase: buildPath = root[:len(root) - len("/build/bin")] break return buildPath - + def prepare_udf_so(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -57,7 +57,7 @@ class TDTestCase: def prepare_data(self): - + tdSql.execute("drop database if exists db ") tdSql.execute("create database if not exists db duration 300") tdSql.execute("use db") @@ -67,7 +67,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -149,7 +149,7 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) - + functions = tdSql.getResult("show functions") function_nums = len(functions) if function_nums == 2: @@ -174,14 +174,14 @@ class TDTestCase: # create aggregate functions tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) - + functions = tdSql.getResult("show functions") function_nums = len(functions) if function_nums == 2: tdLog.info("create two udf functions success ") - + def basic_udf_query(self): - + # scalar functions tdSql.execute("use db ") @@ -255,7 +255,7 @@ class TDTestCase: tdSql.checkData(0,1,165.247614504) tdSql.checkData(0,2,2551.470164435) tdSql.checkData(0,3,2.652476145) - + # # bug for crash when query sub table tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") tdSql.checkData(0,0,378.215547010) @@ -280,7 +280,7 @@ class TDTestCase: tdSql.error("select udf1(num1) , stddev(num1) from tb;") tdSql.error("select udf1(num1) , mode(num1) from tb;") tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") - # stable + # stable tdSql.error("select udf1(c1) , count(c1) from stb1;") tdSql.error("select udf1(c1) , avg(c1) from stb1;") tdSql.error("select udf1(c1) , twa(c1) from stb1;") @@ -301,23 +301,25 @@ class TDTestCase: tdSql.query("select ceil(num1) , min(num1) from tb;") tdSql.checkRows(1) tdSql.query("select udf1(num1) , first(num1) from tb;") - + tdSql.query("select abs(num1) , first(num1) from tb;") - + tdSql.query("select udf1(num1) , last(num1) from tb;") - + tdSql.query("select round(num1) , last(num1) from tb;") - + tdSql.query("select udf1(num1) , top(num1,1) from tb;") tdSql.checkRows(1) tdSql.query("select udf1(num1) , bottom(num1,1) from tb;") tdSql.checkRows(1) - tdSql.error("select udf1(num1) , last_row(num1) from tb;") - - tdSql.error("select round(num1) , last_row(num1) from tb;") - - - # stable + tdSql.query("select udf1(num1) , last_row(num1) from tb;") + tdSql.checkRows(1) + + tdSql.query("select round(num1) , last_row(num1) from tb;") + tdSql.checkRows(1) + + + # stable tdSql.query("select udf1(c1) , max(c1) from stb1;") tdSql.checkRows(1) tdSql.query("select abs(c1) , max(c1) from stb1;") @@ -327,9 +329,9 @@ class TDTestCase: tdSql.query("select floor(c1) , min(c1) from stb1;") tdSql.checkRows(1) tdSql.query("select udf1(c1) , first(c1) from stb1;") - + tdSql.query("select udf1(c1) , last(c1) from stb1;") - + tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;") tdSql.checkRows(1) tdSql.query("select abs(c1) , top(c1 ,1) from stb1;") @@ -339,9 +341,11 @@ class TDTestCase: tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;") tdSql.checkRows(1) - tdSql.error("select udf1(c1) , last_row(c1) from stb1;") - tdSql.error("select ceil(c1) , last_row(c1) from stb1;") - + tdSql.query("select udf1(c1) , last_row(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select ceil(c1) , last_row(c1) from stb1;") + tdSql.checkRows(1) + # regular table with compute functions tdSql.query("select udf1(num1) , abs(num1) from tb;") @@ -349,7 +353,7 @@ class TDTestCase: tdSql.query("select floor(num1) , abs(num1) from tb;") tdSql.checkRows(12) - # # bug need fix + # # bug need fix #tdSql.query("select udf1(num1) , csum(num1) from tb;") #tdSql.checkRows(9) @@ -381,8 +385,8 @@ class TDTestCase: tdSql.checkData(1,0,88) tdSql.checkData(1,1,7) - # bug fix for crash - # order by udf function result + # bug fix for crash + # order by udf function result for _ in range(50): tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") print(tdSql.queryResult) @@ -400,7 +404,7 @@ class TDTestCase: tdSql.checkData(0,1,88) tdSql.checkData(0,2,-99.990000000) tdSql.checkData(0,3,88) - + tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) tdSql.checkData(0,1,0) @@ -428,7 +432,7 @@ class TDTestCase: tdSql.checkData(0,1,168.819430161) tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - # udf functions with group by + # udf functions with group by tdSql.query("select udf1(c1) from ct1 group by c1") tdSql.checkRows(10) tdSql.query("select udf1(c1) from stb1 group by c1") @@ -451,7 +455,7 @@ class TDTestCase: tdSql.query("select udf2(c1) from stb1 group by floor(c1)") tdSql.checkRows(11) - # udf mix with order by + # udf mix with order by tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") tdSql.checkRows(11) @@ -480,7 +484,7 @@ class TDTestCase: tdSql.checkData(0,1,169.661427555) def try_query_sql(self): - udf1_sqls = [ + udf1_sqls = [ "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" , "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" , "select udf1(num1) , max(num1) from tb;" , @@ -507,7 +511,7 @@ class TDTestCase: "select c1,c2, udf1(c1,c2) from stb1 group by c1,c2" , "select num1,num2,num3,udf1(num1,num2,num3) from tb" , "select c1,c6,udf1(c1,c6) from stb1 order by ts" , - "select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;" + "select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;" ] udf2_sqls = ["select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(c1) from stb1 group by 1-udf1(c1)" , @@ -524,7 +528,7 @@ class TDTestCase: "select udf2(c1) from stb1 group by udf1(c1)" , "select udf2(c1) from stb1 group by floor(c1)" , "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" , - + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , @@ -550,7 +554,7 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - # create function without aggregate + # create function without aggregate tdLog.info(" create function with out aggregate ") tdSql.query("drop function udf1 ") @@ -574,8 +578,8 @@ class TDTestCase: tdSql.error(" select test(c1) from stb1 ") tdSql.error(" select test(c1,c6), test(c6) from stb1 ") tdSql.error(" select test(num1,num2), test(num1) from tb ") - - + + def loop_kill_udfd(self): @@ -584,7 +588,7 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - + cfgPath = buildPath + "/../sim/dnode1/cfg" udfdPath = buildPath +'/build/bin/udfd' @@ -595,19 +599,19 @@ class TDTestCase: tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # stop udfd cmds + # stop udfd cmds get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'" processID = subprocess.check_output(get_processID, shell=True).decode("utf-8") stop_udfd = " kill -9 %s" % processID os.system(stop_udfd) time.sleep(2) - + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,169.661427555) tdSql.checkData(0,1,169.661427555) - # # start udfd cmds + # # start udfd cmds # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &" # tdLog.info("start udfd : %s " % start_udfd) @@ -640,19 +644,19 @@ class TDTestCase: tdDnodes.stop(1) tdDnodes.start(1) time.sleep(2) - - + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - - print(" env is ok for all ") + + print(" env is ok for all ") self.prepare_udf_so() self.prepare_data() self.create_udf_function() self.basic_udf_query() self.multi_cols_udf() self.restart_taosd_query_udf() - - + + def stop(self): tdSql.close() From 4f107908db680d2d2436e926c93ac4b71711ed99 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 30 Jun 2022 19:02:43 +0800 Subject: [PATCH 15/24] fix(tmq): reset window --- source/client/src/tmq.c | 3 ++- source/common/src/tmsg.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 23 +++++++++-------------- source/libs/executor/src/executorimpl.c | 10 ++++------ source/libs/executor/src/scanoperator.c | 2 -- tests/script/tsim/tmq/snapshot.sim | 9 +++------ tests/script/tsim/tmq/snapshot1.sim | 6 +++--- 7 files changed, 23 insertions(+), 34 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index aa78649ba7..bea9d215da 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1547,7 +1547,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { } int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { - /*printf("call poll\n");*/ + /*tscDebug("call poll");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { @@ -1708,6 +1708,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { + /*tscDebug("call poll1");*/ void* rspObj; int64_t startTime = taosGetTimestampMs(); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 78eccdf0cd..80402439fc 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5398,9 +5398,9 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { } else if (pVal->type == TMQ_OFFSET__LOG) { snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { - snprintf(buf, maxLen, "offset(snapshot data) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + snprintf(buf, maxLen, "offset(ss data) uid:%ld, ts:%ld", pVal->uid, pVal->ts); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) { - snprintf(buf, maxLen, "offset(snapshot meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + snprintf(buf, maxLen, "offset(ss meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts); } else { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c0d184ee11..d80996b399 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -154,10 +154,10 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con }; tmsgSendRsp(&rsp); - char buf1[50]; - char buf2[50]; - tFormatOffset(buf1, 50, &pRsp->reqOffset); - tFormatOffset(buf2, 50, &pRsp->rspOffset); + char buf1[80]; + char buf2[80]; + tFormatOffset(buf1, 80, &pRsp->reqOffset); + tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s", TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); @@ -238,8 +238,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { STqOffsetVal fetchOffsetNew; // 1.find handle - char buf[50]; - tFormatOffset(buf, 50, &reqOffset); + char buf[80]; + tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch, TD_VID(pTq->pVnode), buf); @@ -360,7 +360,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; metaRsp.reqOffset = pReq->reqOffset.version; - /*tqOffsetResetToLog(&metaR)*/ metaRsp.rspOffset = fetchVer; metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; @@ -380,18 +379,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // 2. get data (rebuild reader if needed) // 3. get new uid and ts - char formatBuf[50]; - tFormatOffset(formatBuf, 50, &dataRsp.reqOffset); - tqInfo("retrieve using snapshot req offset %s", formatBuf); + tqInfo("retrieve using snapshot req offset: uid %ld ts %ld", dataRsp.reqOffset.uid, dataRsp.reqOffset.ts); if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) { ASSERT(0); } // 4. send rsp - if (dataRsp.blockNum != 0) { - if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { - code = -1; - } + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; } } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) { ASSERT(0); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index dd48229849..efa8887ed0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2836,12 +2836,10 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { if (pInfo->lastStatus.uid != uid || pInfo->lastStatus.ts != ts) { tsdbSetTableId(pInfo->dataReader, uid); - SQueryTableDataCond tmpCond = pInfo->cond; - tmpCond.twindows[0] = (STimeWindow){ - .skey = ts, - .ekey = INT64_MAX, - }; - tsdbResetReadHandle(pInfo->dataReader, &tmpCond, 0); + int64_t oldSkey = pInfo->cond.twindows[0].skey; + pInfo->cond.twindows[0].skey = ts; + tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); + pInfo->cond.twindows[0].skey = oldSkey; pInfo->scanTimes = 0; pInfo->curTWinIdx = 0; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a6bf9a12f3..a3af4ab223 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -518,7 +518,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // if scan table by table if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { - // check status while (1) { SSDataBlock* result = doTableScanGroup(pOperator); if (result) { @@ -530,7 +529,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable); - /*pTableInfo->uid */ tsdbSetTableId(pInfo->dataReader, pTableInfo->uid); tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); pInfo->scanTimes = 0; diff --git a/tests/script/tsim/tmq/snapshot.sim b/tests/script/tsim/tmq/snapshot.sim index 5683aaa559..de0468e6f2 100644 --- a/tests/script/tsim/tmq/snapshot.sim +++ b/tests/script/tsim/tmq/snapshot.sim @@ -111,7 +111,7 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = 1 +$expectmsgcnt = 1000000 $expectrowcnt = 100 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) @@ -131,9 +131,6 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi if $data[0][3] != $expectrowcnt then return -1 endi @@ -183,7 +180,7 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = 1 +$expectmsgcnt = 1000000 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb @@ -254,7 +251,7 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb +$expectmsgcnt = 1000000 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb diff --git a/tests/script/tsim/tmq/snapshot1.sim b/tests/script/tsim/tmq/snapshot1.sim index 9226c475cb..d534bb68da 100644 --- a/tests/script/tsim/tmq/snapshot1.sim +++ b/tests/script/tsim/tmq/snapshot1.sim @@ -80,7 +80,7 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfStb +$expectmsgcnt = 1000000 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) @@ -168,7 +168,7 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfCtb +$expectmsgcnt = 1000000 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ctb_function @@ -245,7 +245,7 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfNtb +$expectmsgcnt = 1000000 sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) From 298c4d3e44c3a3a724dcd3e511d4f24550646084 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 30 Jun 2022 20:14:57 +0800 Subject: [PATCH 16/24] tlrucache: fix cap calculation --- source/util/src/tlrucache.c | 194 ++++++++++++++++++++---------------- 1 file changed, 107 insertions(+), 87 deletions(-) diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index b034a6e73e..28e15dc9a3 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -16,24 +16,24 @@ #define _DEFAULT_SOURCE #include "tlrucache.h" #include "os.h" -#include "tdef.h" #include "taoserror.h" -#include "tlog.h" #include "tarray.h" +#include "tdef.h" +#include "tlog.h" -typedef struct SLRUEntry SLRUEntry; +typedef struct SLRUEntry SLRUEntry; typedef struct SLRUEntryTable SLRUEntryTable; typedef struct SLRUCacheShard SLRUCacheShard; -typedef struct SShardedCache SShardedCache; +typedef struct SShardedCache SShardedCache; enum { - TAOS_LRU_IN_CACHE = (1 << 0), // Whether this entry is referenced by the hash table. + TAOS_LRU_IN_CACHE = (1 << 0), // Whether this entry is referenced by the hash table. - TAOS_LRU_IS_HIGH_PRI = (1 << 1), // Whether this entry is high priority entry. + TAOS_LRU_IS_HIGH_PRI = (1 << 1), // Whether this entry is high priority entry. - TAOS_LRU_IN_HIGH_PRI_POOL = (1 << 2), // Whether this entry is in high-pri pool. + TAOS_LRU_IN_HIGH_PRI_POOL = (1 << 2), // Whether this entry is in high-pri pool. - TAOS_LRU_HAS_HIT = (1 << 3), // Whether this entry has had any lookups (hits). + TAOS_LRU_HAS_HIT = (1 << 3), // Whether this entry has had any lookups (hits). }; struct SLRUEntry { @@ -50,18 +50,39 @@ struct SLRUEntry { char keyData[1]; }; -#define TAOS_LRU_ENTRY_IN_CACHE(h) ((h)->flags & TAOS_LRU_IN_CACHE) +#define TAOS_LRU_ENTRY_IN_CACHE(h) ((h)->flags & TAOS_LRU_IN_CACHE) #define TAOS_LRU_ENTRY_IN_HIGH_POOL(h) ((h)->flags & TAOS_LRU_IN_HIGH_PRI_POOL) -#define TAOS_LRU_ENTRY_IS_HIGH_PRI(h) ((h)->flags & TAOS_LRU_IS_HIGH_PRI) -#define TAOS_LRU_ENTRY_HAS_HIT(h) ((h)->flags & TAOS_LRU_HAS_HIT) +#define TAOS_LRU_ENTRY_IS_HIGH_PRI(h) ((h)->flags & TAOS_LRU_IS_HIGH_PRI) +#define TAOS_LRU_ENTRY_HAS_HIT(h) ((h)->flags & TAOS_LRU_HAS_HIT) -#define TAOS_LRU_ENTRY_SET_IN_CACHE(h, inCache) do { if(inCache) {(h)->flags |= TAOS_LRU_IN_CACHE;} else {(h)->flags &= ~TAOS_LRU_IN_CACHE;} } while(0) -#define TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(h, inHigh) do { if(inHigh) {(h)->flags |= TAOS_LRU_IN_HIGH_PRI_POOL;} else {(h)->flags &= ~TAOS_LRU_IN_HIGH_PRI_POOL;} } while(0) -#define TAOS_LRU_ENTRY_SET_PRIORITY(h, priority) do { if(priority == TAOS_LRU_PRIORITY_HIGH) {(h)->flags |= TAOS_LRU_IS_HIGH_PRI;} else {(h)->flags &= ~TAOS_LRU_IS_HIGH_PRI;} } while(0) +#define TAOS_LRU_ENTRY_SET_IN_CACHE(h, inCache) \ + do { \ + if (inCache) { \ + (h)->flags |= TAOS_LRU_IN_CACHE; \ + } else { \ + (h)->flags &= ~TAOS_LRU_IN_CACHE; \ + } \ + } while (0) +#define TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(h, inHigh) \ + do { \ + if (inHigh) { \ + (h)->flags |= TAOS_LRU_IN_HIGH_PRI_POOL; \ + } else { \ + (h)->flags &= ~TAOS_LRU_IN_HIGH_PRI_POOL; \ + } \ + } while (0) +#define TAOS_LRU_ENTRY_SET_PRIORITY(h, priority) \ + do { \ + if (priority == TAOS_LRU_PRIORITY_HIGH) { \ + (h)->flags |= TAOS_LRU_IS_HIGH_PRI; \ + } else { \ + (h)->flags &= ~TAOS_LRU_IS_HIGH_PRI; \ + } \ + } while (0) #define TAOS_LRU_ENTRY_SET_HIT(h) ((h)->flags |= TAOS_LRU_HAS_HIT) #define TAOS_LRU_ENTRY_HAS_REFS(h) ((h)->refs > 0) -#define TAOS_LRU_ENTRY_REF(h) (++(h)->refs) +#define TAOS_LRU_ENTRY_REF(h) (++(h)->refs) static bool taosLRUEntryUnref(SLRUEntry *entry) { assert(entry->refs > 0); @@ -90,7 +111,7 @@ struct SLRUEntryTable { static int taosLRUEntryTableInit(SLRUEntryTable *table, int maxUpperHashBits) { table->lengthBits = 4; - table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry*)); + table->list = taosMemoryCalloc(1 << table->lengthBits, sizeof(SLRUEntry *)); if (!table->list) { return -1; } @@ -125,7 +146,7 @@ static void taosLRUEntryTableCleanup(SLRUEntryTable *table) { taosMemoryFree(table->list); } -static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) { +static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) { SLRUEntry **entry = &table->list[hash >> (32 - table->lengthBits)]; while (*entry && ((*entry)->hash != hash || memcmp(key, (*entry)->keyData, keyLen) != 0)) { entry = &(*entry)->nextHash; @@ -134,7 +155,7 @@ static SLRUEntry **taosLRUEntryTableFindPtr(SLRUEntryTable * table, const void * return entry; } -static void taosLRUEntryTableResize(SLRUEntryTable * table) { +static void taosLRUEntryTableResize(SLRUEntryTable *table) { int lengthBits = table->lengthBits; if (lengthBits >= table->maxLengthBits) { return; @@ -144,9 +165,9 @@ static void taosLRUEntryTableResize(SLRUEntryTable * table) { return; } - uint32_t oldLength = 1 << lengthBits; - int newLengthBits = lengthBits + 1; - SLRUEntry **newList = taosMemoryCalloc(1 << newLengthBits, sizeof(SLRUEntry*)); + uint32_t oldLength = 1 << lengthBits; + int newLengthBits = lengthBits + 1; + SLRUEntry **newList = taosMemoryCalloc(1 << newLengthBits, sizeof(SLRUEntry *)); if (!newList) { return; } @@ -154,8 +175,8 @@ static void taosLRUEntryTableResize(SLRUEntryTable * table) { for (uint32_t i = 0; i < oldLength; ++i) { SLRUEntry *entry = table->list[i]; while (entry) { - SLRUEntry *next = entry->nextHash; - uint32_t hash = entry->hash; + SLRUEntry *next = entry->nextHash; + uint32_t hash = entry->hash; SLRUEntry **ptr = &newList[hash >> (32 - newLengthBits)]; entry->nextHash = *ptr; *ptr = entry; @@ -170,13 +191,13 @@ static void taosLRUEntryTableResize(SLRUEntryTable * table) { table->lengthBits = newLengthBits; } -static SLRUEntry *taosLRUEntryTableLookup(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) { +static SLRUEntry *taosLRUEntryTableLookup(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) { return *taosLRUEntryTableFindPtr(table, key, keyLen, hash); } -static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable * table, SLRUEntry *entry) { +static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable *table, SLRUEntry *entry) { SLRUEntry **ptr = taosLRUEntryTableFindPtr(table, entry->keyData, entry->keyLength, entry->hash); - SLRUEntry *old = *ptr; + SLRUEntry *old = *ptr; entry->nextHash = (old == NULL) ? NULL : old->nextHash; *ptr = entry; if (old == NULL) { @@ -189,9 +210,9 @@ static SLRUEntry *taosLRUEntryTableInsert(SLRUEntryTable * table, SLRUEntry *ent return old; } -static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable * table, const void *key, size_t keyLen, uint32_t hash) { +static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable *table, const void *key, size_t keyLen, uint32_t hash) { SLRUEntry **entry = taosLRUEntryTableFindPtr(table, key, keyLen, hash); - SLRUEntry *result = *entry; + SLRUEntry *result = *entry; if (result) { *entry = result->nextHash; --table->elems; @@ -201,17 +222,17 @@ static SLRUEntry *taosLRUEntryTableRemove(SLRUEntryTable * table, const void *ke } struct SLRUCacheShard { - size_t capacity; - size_t highPriPoolUsage; - bool strictCapacity; - double highPriPoolRatio; - double highPriPoolCapacity; - SLRUEntry lru; - SLRUEntry *lruLowPri; - SLRUEntryTable table; - size_t usage; // Memory size for entries residing in the cache. - size_t lruUsage; // Memory size for entries residing only in the LRU list. - TdThreadMutex mutex; + size_t capacity; + size_t highPriPoolUsage; + bool strictCapacity; + double highPriPoolRatio; + double highPriPoolCapacity; + SLRUEntry lru; + SLRUEntry *lruLowPri; + SLRUEntryTable table; + size_t usage; // Memory size for entries residing in the cache. + size_t lruUsage; // Memory size for entries residing only in the LRU list. + TdThreadMutex mutex; }; #define TAOS_LRU_CACHE_SHARD_HASH32(key, len) (MurmurHash3_32((key), (len))) @@ -231,8 +252,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { assert(e->next == NULL); assert(e->prev == NULL); - if (shard->highPriPoolRatio > 0 - && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) { + if (shard->highPriPoolRatio > 0 && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) { e->next = &shard->lru; e->prev = shard->lru.prev; @@ -248,7 +268,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { e->prev->next = e; e->next->prev = e; - + TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(e, false); shard->lruLowPri = e; } @@ -304,13 +324,13 @@ static void taosLRUCacheShardSetCapacity(SLRUCacheShard *shard, size_t capacity) for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); - taosLRUEntryFree(entry); + taosLRUEntryFree(entry); } taosArrayDestroy(lastReferenceList); } -static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool strict, - double highPriPoolRatio, int maxUpperHashBits) { +static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool strict, double highPriPoolRatio, + int maxUpperHashBits) { if (taosLRUEntryTableInit(&shard->table, maxUpperHashBits) < 0) { return -1; } @@ -341,23 +361,24 @@ static void taosLRUCacheShardCleanup(SLRUCacheShard *shard) { taosLRUEntryTableCleanup(&shard->table); } -static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, bool freeOnFail) { +static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry *e, LRUHandle **handle, + bool freeOnFail) { LRUStatus status = TAOS_LRU_STATUS_OK; - SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); + SArray *lastReferenceList = taosArrayInit(16, POINTER_BYTES); taosThreadMutexLock(&shard->mutex); taosLRUCacheShardEvictLRU(shard, e->totalCharge, lastReferenceList); - + if (shard->usage + e->totalCharge > shard->capacity && (shard->strictCapacity || handle == NULL)) { TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (handle == NULL) { taosArrayPush(lastReferenceList, &e); } else { if (freeOnFail) { - taosMemoryFree(e); + taosMemoryFree(e); - *handle = NULL; + *handle = NULL; } status = TAOS_LRU_STATUS_INCOMPLETE; @@ -371,21 +392,21 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * assert(TAOS_LRU_ENTRY_IN_CACHE(old)); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { - taosLRUCacheShardLRURemove(shard, old); - assert(shard->usage >= old->totalCharge); - shard->usage -= old->totalCharge; + taosLRUCacheShardLRURemove(shard, old); + assert(shard->usage >= old->totalCharge); + shard->usage -= old->totalCharge; - taosArrayPush(lastReferenceList, &old); + taosArrayPush(lastReferenceList, &old); } } if (handle == NULL) { taosLRUCacheShardLRUInsert(shard, e); } else { if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { - TAOS_LRU_ENTRY_REF(e); + TAOS_LRU_ENTRY_REF(e); } - *handle = (LRUHandle*) e; + *handle = (LRUHandle *)e; } } @@ -394,7 +415,7 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); - taosLRUEntryFree(entry); + taosLRUEntryFree(entry); } taosArrayDestroy(lastReferenceList); @@ -402,8 +423,8 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * } static LRUStatus taosLRUCacheShardInsert(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash, - void *value, size_t charge, _taos_lru_deleter_t deleter, - LRUHandle **handle, LRUPriority priority) { + void *value, size_t charge, _taos_lru_deleter_t deleter, LRUHandle **handle, + LRUPriority priority) { SLRUEntry *e = taosMemoryCalloc(1, sizeof(SLRUEntry) - 1 + keyLen); if (!e) { return TAOS_LRU_STATUS_FAIL; @@ -442,7 +463,7 @@ static LRUHandle *taosLRUCacheShardLookup(SLRUCacheShard *shard, const void *key taosThreadMutexUnlock(&shard->mutex); - return (LRUHandle *) e; + return (LRUHandle *)e; } static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_t keyLen, uint32_t hash) { @@ -482,7 +503,7 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); assert(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; - + taosArrayPush(lastReferenceList, &old); } @@ -491,14 +512,14 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { for (int i = 0; i < taosArrayGetSize(lastReferenceList); ++i) { SLRUEntry *entry = taosArrayGetP(lastReferenceList, i); - taosLRUEntryFree(entry); + taosLRUEntryFree(entry); } taosArrayDestroy(lastReferenceList); } static bool taosLRUCacheShardRef(SLRUCacheShard *shard, LRUHandle *handle) { - SLRUEntry *e = (SLRUEntry *) handle; + SLRUEntry *e = (SLRUEntry *)handle; taosThreadMutexLock(&shard->mutex); assert(TAOS_LRU_ENTRY_HAS_REFS(e)); @@ -514,8 +535,8 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b return false; } - SLRUEntry *e = (SLRUEntry *) handle; - bool lastReference = false; + SLRUEntry *e = (SLRUEntry *)handle; + bool lastReference = false; taosThreadMutexLock(&shard->mutex); @@ -537,7 +558,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b assert(shard->usage >= e->totalCharge); shard->usage -= e->totalCharge; } - + taosThreadMutexUnlock(&shard->mutex); if (lastReference) { @@ -549,7 +570,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b static size_t taosLRUCacheShardGetUsage(SLRUCacheShard *shard) { size_t usage = 0; - + taosThreadMutexLock(&shard->mutex); usage = shard->usage; taosThreadMutexUnlock(&shard->mutex); @@ -559,7 +580,7 @@ static size_t taosLRUCacheShardGetUsage(SLRUCacheShard *shard) { static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) { size_t usage = 0; - + taosThreadMutexLock(&shard->mutex); assert(shard->usage >= shard->lruUsage); @@ -579,11 +600,11 @@ static void taosLRUCacheShardSetStrictCapacity(SLRUCacheShard *shard, bool stric } struct SShardedCache { - uint32_t shardMask; - TdThreadMutex capacityMutex; - size_t capacity; - bool strictCapacity; - uint64_t lastId; // atomic var for last id + uint32_t shardMask; + TdThreadMutex capacityMutex; + size_t capacity; + bool strictCapacity; + uint64_t lastId; // atomic var for last id }; struct SLRUCache { @@ -593,7 +614,7 @@ struct SLRUCache { }; static int getDefaultCacheShardBits(size_t capacity) { - int numShardBits = 0; + int numShardBits = 0; size_t minShardSize = 512 * 1024; size_t numShards = capacity / minShardSize; while (numShards >>= 1) { @@ -621,7 +642,7 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo numShardBits = getDefaultCacheShardBits(capacity); } - int numShards = 1 << numShardBits; + int numShards = 1 << numShardBits; cache->shards = taosMemoryCalloc(numShards, sizeof(SLRUCacheShard)); if (!cache->shards) { taosMemoryFree(cache); @@ -629,7 +650,7 @@ SLRUCache *taosLRUCacheInit(size_t capacity, int numShardBits, double highPriPoo return NULL; } - bool strictCapacity = 1; + bool strictCapacity = 1; size_t perShard = (capacity + (numShards - 1)) / numShards; for (int i = 0; i < numShards; ++i) { taosLRUCacheShardInit(&cache->shards[i], perShard, strictCapacity, highPriPoolRatio, 32 - numShardBits); @@ -653,7 +674,7 @@ void taosLRUCacheCleanup(SLRUCache *cache) { int numShards = cache->numShards; assert(numShards > 0); for (int i = 0; i < numShards; ++i) { - taosLRUCacheShardCleanup(&cache->shards[i]); + taosLRUCacheShardCleanup(&cache->shards[i]); } taosMemoryFree(cache->shards); cache->shards = 0; @@ -666,11 +687,12 @@ void taosLRUCacheCleanup(SLRUCache *cache) { } LRUStatus taosLRUCacheInsert(SLRUCache *cache, const void *key, size_t keyLen, void *value, size_t charge, - _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority) { + _taos_lru_deleter_t deleter, LRUHandle **handle, LRUPriority priority) { uint32_t hash = TAOS_LRU_CACHE_SHARD_HASH32(key, keyLen); uint32_t shardIndex = hash & cache->shardedCache.shardMask; - return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle, priority); + return taosLRUCacheShardInsert(&cache->shards[shardIndex], key, keyLen, hash, value, charge, deleter, handle, + priority); } LRUHandle *taosLRUCacheLookup(SLRUCache *cache, const void *key, size_t keyLen) { @@ -699,7 +721,7 @@ bool taosLRUCacheRef(SLRUCache *cache, LRUHandle *handle) { return false; } - uint32_t hash = ((SLRUEntry *) handle)->hash; + uint32_t hash = ((SLRUEntry *)handle)->hash; uint32_t shardIndex = hash & cache->shardedCache.shardMask; return taosLRUCacheShardRef(&cache->shards[shardIndex], handle); @@ -710,15 +732,13 @@ bool taosLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRe return false; } - uint32_t hash = ((SLRUEntry *) handle)->hash; + uint32_t hash = ((SLRUEntry *)handle)->hash; uint32_t shardIndex = hash & cache->shardedCache.shardMask; return taosLRUCacheShardRelease(&cache->shards[shardIndex], handle, eraseIfLastRef); } -void* taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle) { - return ((SLRUEntry*) handle)->value; -} +void *taosLRUCacheValue(SLRUCache *cache, LRUHandle *handle) { return ((SLRUEntry *)handle)->value; } size_t taosLRUCacheGetUsage(SLRUCache *cache) { size_t usage = 0; @@ -742,7 +762,7 @@ size_t taosLRUCacheGetPinnedUsage(SLRUCache *cache) { void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity) { uint32_t numShards = cache->numShards; - size_t perShard = (capacity + (numShards = 1)) / numShards; + size_t perShard = (capacity + (numShards - 1)) / numShards; taosThreadMutexLock(&cache->shardedCache.capacityMutex); @@ -751,7 +771,7 @@ void taosLRUCacheSetCapacity(SLRUCache *cache, size_t capacity) { } cache->shardedCache.capacity = capacity; - + taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); } @@ -777,7 +797,7 @@ void taosLRUCacheSetStrictCapacity(SLRUCache *cache, bool strict) { } cache->shardedCache.strictCapacity = strict; - + taosThreadMutexUnlock(&cache->shardedCache.capacityMutex); } From 10dcf571ce352dca0e03b680df7e06820ecccd9e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Jun 2022 20:20:27 +0800 Subject: [PATCH 17/24] test: add test case for tmq --- tests/pytest/util/common.py | 14 +- tests/system-test/7-tmq/tmqAlterSchema.py | 259 ++++++++++++++++++++++ tests/system-test/7-tmq/tmqCommon.py | 63 ++++++ tests/system-test/fulltest.sh | 1 + 4 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqAlterSchema.py diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 998e0248ab..6425e7acfa 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -535,6 +535,7 @@ class TDCom: ts_value = self.genTs()[0] column_value_list = list() + column_value_list.append(ts_value) if column_elm_list is None: column_value_list = list(map(lambda i: self.gen_random_type_value(i, self.default_varchar_length, self.default_varchar_datatype, self.default_nchar_length, self.default_nchar_datatype), self.full_type_list)) else: @@ -551,7 +552,7 @@ class TDCom: column_value_list.append(self.gen_random_type_value(column_elm["type"], "", "", "", "")) else: continue - column_value_list = [self.ts_value] + self.column_value_list + # column_value_list = [self.ts_value] + self.column_value_list return column_value_list def create_stable(self, tsql, dbname=None, stbname="stb", column_elm_list=None, tag_elm_list=None, @@ -640,7 +641,16 @@ class TDCom: else: for num in range(count): column_value_list = self.gen_column_value_list(column_ele_list, f'{start_ts_value}+{num}s') - column_value_str = ", ".join(str(v) for v in column_value_list) + # column_value_str = ", ".join(str(v) for v in column_value_list) + column_value_str = '' + idx = 0 + for column_value in column_value_list: + if isinstance(column_value, str) and idx != 0: + column_value_str += f'"{column_value}", ' + else: + column_value_str += f'{column_value}, ' + idx += 1 + column_value_str = column_value_str.rstrip()[:-1] insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});' tsql.execute(insert_sql) def getOneRow(self, location, containElm): diff --git a/tests/system-test/7-tmq/tmqAlterSchema.py b/tests/system-test/7-tmq/tmqAlterSchema.py new file mode 100644 index 0000000000..a2e20990d9 --- /dev/null +++ b/tests/system-test/7-tmq/tmqAlterSchema.py @@ -0,0 +1,259 @@ + +from ntpath import join +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.dnodes = 5 + self.mnodes = 3 + self.idIndex = 0 + self.roleIndex = 2 + self.mnodeStatusIndex = 3 + self.mnodeEpIndex = 1 + self.dnodeStatusIndex = 4 + self.mnodeCheckCnt = 10 + self.host = socket.gethostname() + self.startPort = 6030 + self.portStep = 100 + self.dnodeOfLeader = 0 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: topic: select * from stb, while consume, add column int-A/bianry-B/float-C, and then modify B, drop C") + tdLog.printNoPrefix("add tag int-A/bianry-B/float-C, and then rename A, modify B, drop C, set t2") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}, {'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1'] + expectRowsList = [] + queryStringList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + # tdLog.info("async insert data") + # pThread = tmqCom.asyncInsertData(paraDict) + tmqCom.insert_data_2(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"]) + + tdLog.info("create topics from stb with filter") + queryStringList.append("select * from %s.%s" %(paraDict['dbName'], paraDict['stbName'])) + sqlString = "create topic %s as %s" %(topicNameList[0], queryStringList[0]) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryStringList[0]) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + dstFile = tmqCom.getResultFileByTaosShell(consumerId, queryStringList[0]) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the notify info of start consume, then alter schema") + tmqCom.getStartConsumeNotifyFromTmqsim() + + # add column double-A/bianry-B/double-C, and then modify B, drop C + sqlString = "alter table %s.%s add column newc1 double"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s add column newc2 binary(16)"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s add column newc3 double"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s modify column newc2 binary(32)"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s drop column newc3"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + # add tag double-A/bianry-B/double-C, and then rename A, modify B, drop C, set t1 + sqlString = "alter table %s.%s add tag newt1 double"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s add tag newt2 binary(16)"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s add tag newt3 double"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s rename tag newt1 newt1n"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s modify tag newt2 binary(32)"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s drop tag newt3"%(paraDict["dbName"],paraDict['stbName']) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s0 set tag newt2='new tag'"%(paraDict["dbName"],paraDict['ctbPrefix']) + tdSql.execute(sqlString) + + tdLog.info("check the consume result") + tdSql.query(queryStringList[0]) + expectRowsList.append(tdSql.getRows()) + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + tdLog.info("expect consume rows: %d"%(expectRowsList[0])) + tdLog.info("act consume rows: %d"%(resultList[0])) + + if expectRowsList[0] != resultList[0]: + tdLog.exit("0 tmq consume rows error!") + + tmqCom.checkTmqConsumeFileContent(consumerId, dstFile) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: topic: select * from ntb, while consume, add column int-A/bianry-B/float-C, and then rename A, modify B, drop C") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':2}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + ntbName = 'ntb' + + topicNameList = ['topic1'] + expectRowsList = [] + queryStringList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ntb") + tdCom.create_table(tdSql, dbname=paraDict["dbName"],tbname=ntbName,column_elm_list=paraDict['colSchema'],count=1) + tdLog.info("start insert data ....") + # pThread = tmqCom.asyncInsertData(paraDict) + tdCom.insert_rows(tdSql, dbname=paraDict["dbName"], tbname=ntbName, column_ele_list=paraDict['colSchema'], start_ts_value=paraDict["startTs"], count=paraDict["rowsPerTbl"]) + tdLog.info("insert data end") + + tdLog.info("create topics from ntb with filter") + queryStringList.append("select * from %s.%s" %(paraDict['dbName'], ntbName)) + sqlString = "create topic %s as %s" %(topicNameList[0], queryStringList[0]) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryStringList[0]) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + dstFile = tmqCom.getResultFileByTaosShell(consumerId, queryStringList[0]) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the notify info of start consume, then alter schema") + tmqCom.getStartConsumeNotifyFromTmqsim() + + # add column double-A/bianry-B/double-C, and then rename A, modify B, drop C + sqlString = "alter table %s.%s add column newc1 double"%(paraDict["dbName"],ntbName) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s add column newc2 binary(16)"%(paraDict["dbName"],ntbName) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s add column newc3 double"%(paraDict["dbName"],ntbName) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s rename column newc1 newc1n"%(paraDict["dbName"],ntbName) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s modify column newc2 binary(32)"%(paraDict["dbName"],ntbName) + tdSql.execute(sqlString) + sqlString = "alter table %s.%s drop column newc3"%(paraDict["dbName"],ntbName) + tdSql.execute(sqlString) + + tdLog.info("check the consume result") + tdSql.query(queryStringList[0]) + expectRowsList.append(tdSql.getRows()) + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + tdLog.info("expect consume rows: %d"%(expectRowsList[0])) + tdLog.info("act consume rows: %d"%(resultList[0])) + + if expectRowsList[0] != resultList[0]: + tdLog.exit("0 tmq consume rows error!") + + tmqCom.checkTmqConsumeFileContent(consumerId, dstFile) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + self.tmqCase1() + self.tmqCase2() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index b545340153..04adb93b8a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -383,6 +383,69 @@ class TMQCom: pThread.start() return pThread + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + lines = 0 + while True: + dst = queryFile.readline() + src = consumeFile.readline() + lines += 1 + if dst: + if dst != src: + tdLog.info("src row: %s"%src) + tdLog.info("dst row: %s"%dst) + tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines)) + else: + break + return + + def getResultFileByTaosShell(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + return dstFile + + def checkTmqConsumeFileContent(self, consumerId, dstFile): + cfgPath = tdCom.getClientCfgPath() + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + lines = 0 + while True: + dst = queryFile.readline() + src = consumeFile.readline() + lines += 1 + if dst: + if dst != src: + tdLog.info("src row: %s"%src) + tdLog.info("dst row: %s"%dst) + tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines)) + else: + break + return + def close(self): self.cursor.close() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 44a3afcf73..a4e191b36a 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -153,3 +153,4 @@ python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py python3 ./test.py -f 7-tmq/tmqShow.py +python3 ./test.py -f 7-tmq/tmqAlterSchema.py From 68cfa54e26f0eb2fdec8fd0201fa839d6efd8c43 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 30 Jun 2022 20:21:24 +0800 Subject: [PATCH 18/24] feat(stream): ignore close window --- source/libs/executor/src/timewindowoperator.c | 63 +++++++++++++++---- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 95b16589ee..66900fb7aa 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1246,6 +1246,9 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS pCtx[i].fpSet.init(&pCtx[i], pResInfo); } } + SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId); + setBufPageDirty(bufPage, true); + releaseBufPage(pResultBuf, bufPage); } bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, @@ -3171,6 +3174,10 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex); step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL); ASSERT(isInWindow(pCurWin, tsCols[i], gap)); + if (pCurWin->pos.pageId == -1) { + // window has been closed. + continue; + } doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); if (result) { taosArrayPush(result, pCurWin); @@ -3246,12 +3253,12 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo); - SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); + SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); + releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage); continue; + } else if (!pChWin->isClosed) { + break; } - break; } } SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId); @@ -3265,7 +3272,8 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*); SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; } SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } -int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn) { +int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, + SArray* pClosed, __get_win_info_ fn, bool delete) { // Todo(liuyao) save window to tdb void** pIte = NULL; size_t keyLen = 0; @@ -3279,10 +3287,18 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra if (isCloseWindow(&pSeWin->win, pTwSup)) { if (!pSeWin->isClosed) { pSeWin->isClosed = true; - if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed); + if (code != TSDB_CODE_SUCCESS) { + return code; + } pSeWin->isOutput = true; } + if (delete) { + taosArrayRemove(pWins, i); + i--; + size = taosArrayGetSize(pWins); + } } continue; } @@ -3292,6 +3308,16 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra return TSDB_CODE_SUCCESS; } +static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete) { + int32_t size = taosArrayGetSize(pChildren); + for (int32_t i = 0; i < size; i++) { + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; + pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); + closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete); + } +} + int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) { void** pIte = NULL; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -3339,6 +3365,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pBlock == NULL) { break; } + printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "Final Session Recv" : "Single Session Recv"); if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); @@ -3385,7 +3412,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, + getResWinForSession, pInfo->ignoreCloseWindow); + closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow); copyUpdateResult(pStUpdated, pUpdated); taosHashCleanup(pStUpdated); @@ -3437,10 +3466,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "Semi Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0) { + if (pBInfo->pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; if (pInfo->pUpdateRes->info.rows == 0) { // semi interval operator clear disk buffer @@ -3449,9 +3479,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } // process the rest of the data pOperator->status = OP_OPENED; + printDataBlock(pInfo->pUpdateRes, "Semi Session"); return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + printDataBlock(pBInfo->pRes, "Semi Session"); + return pBInfo->pRes; } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -3495,21 +3527,24 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "Semi Session"); return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0) { + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; if (pInfo->pUpdateRes->info.rows == 0) { return NULL; } // process the rest of the data pOperator->status = OP_OPENED; + printDataBlock(pInfo->pUpdateRes, "Semi Session"); return pInfo->pUpdateRes; } + printDataBlock(pBInfo->pRes, "Semi Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3867,7 +3902,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, + getResWinForState, pInfo->ignoreCloseWindow); + closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow); copyUpdateResult(pSeUpdated, pUpdated); taosHashCleanup(pSeUpdated); From 6883aeb33c5c040dfac86ed8282ce859f89c9b40 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Jun 2022 21:25:16 +0800 Subject: [PATCH 19/24] test: add test case --- tests/test/c/tmqSim.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 7d57ef0781..542c37b7c9 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -633,8 +633,9 @@ void loop_consume(SThreadInfo* pInfo) { } } - uint64_t lastPrintTime = taosGetTimestampMs(); - uint64_t startTs = taosGetTimestampMs(); + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); while (running) { @@ -647,20 +648,22 @@ void loop_consume(SThreadInfo* pInfo) { taos_free_result(tmqMsg); totalMsgs++; - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 10 * 1000) { - taosFprintfFile(g_fp, "consumer id %d has currently poll total msgs: %" PRId64 "\n", pInfo->consumerId, - totalMsgs); - lastPrintTime = currentPrintTime; - } - + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + taosFprintfFile(g_fp, + "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0/(currentPrintTime - lastPrintTime)); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + } + if (0 == once_flag) { once_flag = 1; notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); } - if (totalRows >= pInfo->expectMsgCnt) { + if ((totalRows >= pInfo->expectMsgCnt) || (totalMsgs >= pInfo->expectMsgCnt)) { char tmpString[128]; taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString)); break; @@ -671,7 +674,7 @@ void loop_consume(SThreadInfo* pInfo) { break; } } - + if (0 == running) { taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); } @@ -881,11 +884,11 @@ int main(int32_t argc, char* argv[]) { int64_t t = end - start; if (0 == t) t = 1; - + double tInMs = (double)t / 1000000.0; taosFprintfFile(g_fp, - "Spent %.4f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.2f msgs/second\n\n", - tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp); From 126d884ab3eb8ae16772f54b25840e0c205a20b6 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Jun 2022 21:32:08 +0800 Subject: [PATCH 20/24] docs: use normal doc style (#14411) --- docs/en/14-reference/04-taosadapter.md | 4 ++-- docs/zh/14-reference/04-taosadapter.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index 3264124655..cad229c32d 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -206,8 +206,8 @@ Note: InfluxDB token authorization is not supported at present. Only Basic autho You can use any client that supports the http protocol to access the RESTful interface address `http://:6041/` to write data in OpenTSDB compatible format to TDengine. ```text -/opentsdb/v1/put/json/:db -/opentsdb/v1/put/telnet/:db +/opentsdb/v1/put/json/ +/opentsdb/v1/put/telnet/ ``` ### collectd diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index 6e259391d4..42bc51a6d3 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -207,8 +207,8 @@ AllowWebSockets 您可以使用任何支持 http 协议的客户端访问 Restful 接口地址 `http://:6041/` 来写入 OpenTSDB 兼容格式的数据到 TDengine。EndPoint 如下： ```text -/opentsdb/v1/put/json/:db -/opentsdb/v1/put/telnet/:db +/opentsdb/v1/put/json/ +/opentsdb/v1/put/telnet/ ``` ### collectd From 7a65589525d7ff10446292492e0a07e13fb8a82b Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 1 Jul 2022 10:15:05 +0800 Subject: [PATCH 21/24] feat(wal): add api to check log existance --- include/libs/wal/wal.h | 2 ++ source/client/src/tmq.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 6 +----- source/libs/wal/src/walMeta.c | 4 ++++ tests/test/c/tmqSim.c | 6 +++++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 92701db2ad..eb5db9d639 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -210,6 +210,8 @@ void walCloseRef(SWalRef *); int32_t walRefVer(SWalRef *, int64_t ver); int32_t walUnrefVer(SWal *); +bool walLogExist(SWal *, int64_t ver); + // lifecycle check bool walIsEmpty(SWal *); int64_t walGetFirstVer(SWal *); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index bea9d215da..efc1e3a082 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1609,8 +1609,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int64_t transporterId = 0; /*printf("send poll\n");*/ - char offsetFormatBuf[50]; - tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew); + char offsetFormatBuf[80]; + tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d80996b399..8ef66d3ef6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -276,7 +276,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - if (pReq->useSnapshot) { + if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { if (!pHandle->fetchMeta) { tqOffsetResetToData(&fetchOffsetNew, 0, 0); } else { @@ -375,10 +375,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosMemoryFree(pHeadWithCkSum); } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { - // 1. set uid and ts - // 2. get data (rebuild reader if needed) - // 3. get new uid and ts - tqInfo("retrieve using snapshot req offset: uid %ld ts %ld", dataRsp.reqOffset.uid, dataRsp.reqOffset.ts); if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) { ASSERT(0); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 4150fe6d1b..342ab7b152 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -19,6 +19,10 @@ #include "tref.h" #include "walInt.h" +bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) { + return !walIsEmpty(pWal) && walGetFirstVer(pWal) <= ver && walGetLastVer(pWal) >= ver; +} + bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; } int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 07fdcc2415..5cf38f1e5b 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -36,7 +36,11 @@ #define MAX_CONSUMER_THREAD_CNT (16) #define MAX_VGROUP_CNT (32) -typedef enum { NOTIFY_CMD_START_CONSUM, NOTIFY_CMD_START_COMMIT, NOTIFY_CMD_ID_BUTT } NOTIFY_CMD_ID; +typedef enum { + NOTIFY_CMD_START_CONSUM, + NOTIFY_CMD_START_COMMIT, + NOTIFY_CMD_ID_BUTT, +} NOTIFY_CMD_ID; typedef struct { TdThread thread; From 501254219baacab51abf7929238168c8a97141d8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 1 Jul 2022 10:27:50 +0800 Subject: [PATCH 22/24] test:add test case for tmq --- tests/system-test/7-tmq/tmqCommon.py | 6 +- tests/system-test/7-tmq/tmqConsFromTsdb.py | 107 +++++++++++++++++++++ 2 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqConsFromTsdb.py diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 04adb93b8a..f8488aee49 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -93,7 +93,7 @@ class TMQCom: return resultList - def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0): + def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0,snapshot=0): buildPath = tdCom.getBuildPath() cfgPath = tdCom.getClientCfgPath() if valgrind == 1: @@ -109,7 +109,7 @@ class TMQCom: os.system(shellCmd) processorName = processorNameNew shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += " -y %d -d %s -g %d -r %d -w %s -e %d "%(pollDelay, dbName, showMsg, showRow, cdbName, snapshot) shellCmd += "> nul 2>&1 &" else: processorName = buildPath + '/build/bin/tmq_sim' @@ -119,7 +119,7 @@ class TMQCom: os.system(shellCmd) processorName = processorNameNew shellCmd = 'nohup ' + processorName + ' -c ' + cfgPath - shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += " -y %d -d %s -g %d -r %d -w %s -e %d "%(pollDelay, dbName, showMsg, showRow, cdbName, snapshot) shellCmd += "> /dev/null 2>&1 &" tdLog.info(shellCmd) os.system(shellCmd) diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb.py b/tests/system-test/7-tmq/tmqConsFromTsdb.py new file mode 100644 index 0000000000..8629ab3031 --- /dev/null +++ b/tests/system-test/7-tmq/tmqConsFromTsdb.py @@ -0,0 +1,107 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdDnodes.stop(1) + time.sleep(2) + tdDnodes.start(1) + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + # sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + tmqCom.checkFileContent(consumerId, queryString) + + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ba265d51b8d5d59267bdfe41157846248224544f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 1 Jul 2022 10:29:37 +0800 Subject: [PATCH 23/24] fix: fix stream crash issue --- source/common/src/tdatablock.c | 2 +- source/libs/scalar/src/scalar.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7b142e3053..2918e44e7b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1218,7 +1218,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); - if (pSrc->pData == NULL) { + if (pSrc->pData == NULL && (!IS_VAR_DATA_TYPE(pSrc->info.type))) { continue; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cef8ff7075..8e532df3b0 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -287,7 +287,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC int32_t code = 0; if (NULL == pParamList) { if (ctx->pBlockList) { - SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0); + SSDataBlock *pBlock = taosArrayGetP(ctx->pBlockList, 0); *rowNum = pBlock->info.rows; } else { *rowNum = 1; From 4e8f3db5964095998813bd9c04a5c59af707432c Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 1 Jul 2022 11:54:21 +0800 Subject: [PATCH 24/24] os: replace win qsort func --- include/os/osMath.h | 7 ++++ source/common/src/tdatablock.c | 2 +- source/common/src/tdataformat.c | 4 +-- source/libs/executor/src/executil.c | 2 +- source/libs/function/src/taggfunction.c | 4 +-- source/libs/function/src/tpercentile.c | 2 +- source/libs/parser/src/parInsert.c | 8 ++--- source/libs/parser/src/parInsertData.c | 4 +-- source/libs/scalar/src/filter.c | 4 +-- source/os/src/osMath.c | 47 +++++++++++++++++++++++++ source/util/src/tarray.c | 4 +-- source/util/src/tdigest.c | 2 +- source/util/src/terror.c | 2 +- 13 files changed, 73 insertions(+), 19 deletions(-) create mode 100644 source/os/src/osMath.c diff --git a/include/os/osMath.h b/include/os/osMath.h index e13c32422e..b0c75f4dd7 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -60,6 +60,13 @@ extern "C" { }) #endif +#ifndef __COMPAR_FN_T +#define __COMPAR_FN_T +typedef int32_t (*__compar_fn_t)(const void *, const void *); +#endif + +void taosSort(void* arr, int64_t sz, int64_t width, __compar_fn_t compar); + #ifdef __cplusplus } #endif diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7b142e3053..0ec63e90af 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -892,7 +892,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { int64_t p0 = taosGetTimestampUs(); __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order); - qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); + taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); int64_t p1 = taosGetTimestampUs(); uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 7c1b31b6e4..83ae442ae7 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -931,9 +931,9 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { // sort if (isJson) { - qsort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn); + taosSort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn); } else { - qsort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn); + taosSort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn); } // get size diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2561516f6a..415df595e9 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -118,7 +118,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) { __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc; - qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); + taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); } pGroupResInfo->index = 0; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 47ae07f823..c8998b9d94 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -1887,10 +1887,10 @@ static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) { __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); + taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); + taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } GET_TRUE_DATA_TYPE(); diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index b40dd8b78d..057d2bc7dc 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -44,7 +44,7 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) offset += (int32_t)(pg->num * pMemBucket->bytes); } - qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); + taosSort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn); return buffer; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index b97f1df38b..a76640d0b5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -790,11 +790,11 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } - qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } - qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } if (pColList->numOfCols > pColList->numOfBound) { @@ -2232,11 +2232,11 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS pColIdx[i].schemaColIdx = pColList->boundColumns[i]; pColIdx[i].boundIdx = i; } - qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); + taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), schemaIdxCompar); for (col_id_t i = 0; i < pColList->numOfBound; ++i) { pColIdx[i].finalIdx = i; } - qsort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); + taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), boundIdxCompar); } if (pColList->numOfCols > pColList->numOfBound) { diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 56301c072c..84bcef7185 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -295,7 +295,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) { char* pBlockData = pBlocks->data; // todo. qsort is unstable, if timestamp is same, should get the last one - qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); + taosSort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); int32_t i = 0; int32_t j = 1; @@ -365,7 +365,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; // todo. qsort is unstable, if timestamp is same, should get the last one - qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); + taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; int32_t i = 0; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b8bcae161b..9897acfe1a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2059,7 +2059,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } if (colIdxi > 1) { - qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + taosSort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); } for (uint32_t l = 0; l < colIdxi; ++l) { @@ -2294,7 +2294,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR return TSDB_CODE_SUCCESS; } - qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); + taosSort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); int32_t pEnd = 0, cStart = 0, cEnd = 0; uint32_t pColNum = 0, cColNum = 0; diff --git a/source/os/src/osMath.c b/source/os/src/osMath.c new file mode 100644 index 0000000000..98cd63a831 --- /dev/null +++ b/source/os/src/osMath.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#define _DEFAULT_SOURCE +#include "os.h" + +#ifdef WINDOWS +void swapStr(char* j, char* J, int width) { + int i; + char tmp; + for (i = 0; i < width; i++) { + tmp = *j; + *j = *J; + *J = tmp; + j++; + J++; + } +} +#endif + +void taosSort(void* arr, int64_t sz, int64_t width, __compar_fn_t compar) { +#ifdef WINDOWS + int64_t i, j; + for (i = 0; i < sz - 1; i++) { + for (j = 0; j < sz - 1 - i; j++) { + if (compar((char*)arr + j * width, (char*)arr + (j + 1) * width) > 0.00) { + swapStr((char*)arr + j * width, (char*)arr + (j + 1) * width, width); + } + } + } +#else + qsort(arr, sz, width, compar); +#endif +} diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index ec90e5a9b9..23e79da948 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -373,7 +373,7 @@ void taosArraySort(SArray* pArray, __compar_fn_t compar) { assert(pArray != NULL); assert(compar != NULL); - qsort(pArray->pData, pArray->size, pArray->elemSize, compar); + taosSort(pArray->pData, pArray->size, pArray->elemSize, compar); } void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) { @@ -390,7 +390,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { assert(pArray != NULL); - qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); + taosSort(pArray->pData, pArray->size, pArray->elemSize, comparFn); } char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int32_t flags) { diff --git a/source/util/src/tdigest.c b/source/util/src/tdigest.c index 56b113fd8f..a722cfeee2 100644 --- a/source/util/src/tdigest.c +++ b/source/util/src/tdigest.c @@ -124,7 +124,7 @@ void tdigestCompress(TDigest *t) { t->num_buffered_pts = 0; t->total_weight += unmerged_weight; - qsort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid); + taosSort(unmerged_centroids, num_unmerged, sizeof(SCentroid), cmpCentroid); memset(&args, 0, sizeof(SMergeArgs)); args.centroids = (SCentroid*)taosMemoryMalloc((size_t)(sizeof(SCentroid) * t->size)); memset(args.centroids, 0, (size_t)(sizeof(SCentroid) * t->size)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3ab307dc3f..58de4970a0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -614,7 +614,7 @@ static int32_t taosCompareTaosError(const void* a, const void* b) { static TdThreadOnce tsErrorInit = PTHREAD_ONCE_INIT; static void tsSortError(void) { - qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); + taosSort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); } const char* tstrerror(int32_t err) {