From 515109088f42922114555e1045d3c5869cd7d6e5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 27 Jun 2022 15:08:13 +0800 Subject: [PATCH 01/18] feat(query): support mode function --- source/libs/function/inc/builtinsimpl.h | 5 +++++ source/libs/function/src/builtins.c | 24 +++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 0820e884ce..f7e22cb151 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -191,6 +191,11 @@ bool getUniqueFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool uniqueFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t uniqueFunction(SqlFunctionCtx *pCtx); +bool getModeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool modeFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t modeFunction(SqlFunctionCtx *pCtx); +int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t twaFunction(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index d41bc89a5f..c8e39db90c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1045,20 +1045,28 @@ static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int3 return translateFirstLastImpl(pFunc, pErrBuf, len, false); } -static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { +static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isUnique) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (!nodesExprHasColumn(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of UNIQUE must contain columns"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns", isUnique ? "UNIQUE" : "MODE"); } pFunc->node.resType = ((SExprNode*)pPara)->resType; return TSDB_CODE_SUCCESS; } +static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateUniqueMode(pFunc, pErrBuf, len, true); +} + +static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateUniqueMode(pFunc, pErrBuf, len, false); +} + static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); if (numOfParams == 0 || numOfParams > 2) { @@ -2109,7 +2117,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, @@ -2117,6 +2125,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = uniqueFunction, .finalizeFunc = NULL }, + { + .name = "mode", + .type = FUNCTION_TYPE_MODE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateMode, + .getEnvFunc = getUniqueFuncEnv, + .initFunc = uniqueFunctionSetup, + .processFunc = uniqueFunction, + .finalizeFunc = functionFinalize, + }, { .name = "abs", .type = FUNCTION_TYPE_ABS, From 8088b67dc905dc87da0479620b02890368c8acc8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 27 Jun 2022 19:07:37 +0800 Subject: [PATCH 02/18] feat: add merge projects optimization --- source/libs/planner/src/planOptimizer.c | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f1cad6010c..8420f40ec9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1235,6 +1235,45 @@ static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo return eliminateSetOpOptimizeImpl(pCxt, pLogicSubplan, pSetOpNode); } +//=================================================================================================================== +// merge projects +static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + SLogicNode *pChild = (SLogicNode*)nodesListGetNode(pNode->pChildren, 0); + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pChild) || 1 < LIST_LENGTH(pChild->pChildren) || + NULL != pChild->pConditions || NULL != pNode->pLimit || NULL != pNode->pSlimit) { + return false; + } + return true; +} + +typedef struct SMergeProjectionsContext { + +} SMergeProjectionsContext; + +static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { + return DEAL_RES_CONTINUE; +} + +static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pProjectNode) { + SProjectLogicNode* pProject = (SProjectLogicNode*)pProjectNode; + SProjectLogicNode* pChild = (SProjectLogicNode*)nodesListGetNode(pProjectNode->pChildren, 0); + SMergeProjectionsContext cxt = {}; + nodesRewriteExprs(pChild->pProjections, mergeProjectionsExpr, &cxt); + return TSDB_CODE_SUCCESS; +} + +static int32_t mergeProjectsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SLogicNode* pProjectNode = optFindPossibleNode(pLogicSubplan->pNode, mergeProjectsMayBeOptimized); + if (NULL == pProjectNode) { + return TSDB_CODE_SUCCESS; + } + + return mergeProjectsOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, @@ -1242,8 +1281,11 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "MergeProjects", .optimizeFunc = mergeProjectsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize} + {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, + {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}, }; // clang-format on From f9e80a8e2b458e6c1b626294a370c042c113e0f7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 27 Jun 2022 19:41:13 +0800 Subject: [PATCH 03/18] fix(query): add mode function --- include/util/taoserror.h | 4 +- source/libs/function/src/builtins.c | 8 +- source/libs/function/src/builtinsimpl.c | 104 ++++++++++++++++++++++-- 3 files changed, 104 insertions(+), 12 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8af5945300..8565eea63b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -25,7 +25,7 @@ extern "C" { // clang-format off #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) - + #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) @@ -35,7 +35,7 @@ const char* terrstr(); int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) - + #define TSDB_CODE_SUCCESS 0 #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c8e39db90c..699a050aea 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2130,10 +2130,10 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_MODE, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateMode, - .getEnvFunc = getUniqueFuncEnv, - .initFunc = uniqueFunctionSetup, - .processFunc = uniqueFunction, - .finalizeFunc = functionFinalize, + .getEnvFunc = getModeFuncEnv, + .initFunc = modeFunctionSetup, + .processFunc = modeFunction, + .finalizeFunc = modeFinalize, }, { .name = "abs", diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c4d3a26ab4..ca663ff809 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -32,6 +32,7 @@ #define TAIL_MAX_OFFSET 100 #define UNIQUE_MAX_RESULT_SIZE (1024 * 1024 * 10) +#define MODE_MAX_RESULT_SIZE UNIQUE_MAX_RESULT_SIZE #define HLL_BUCKET_BITS 14 // The bits of the bucket #define HLL_DATA_BITS (64 - HLL_BUCKET_BITS) @@ -246,6 +247,19 @@ typedef struct SUniqueInfo { char pItems[]; } SUniqueInfo; +typedef struct SModeItem { + int64_t count; + char data[]; +} SModeItem; + +typedef struct SModeInfo { + int32_t numOfPoints; + uint8_t colType; + int16_t colBytes; + SHashObj* pHash; + char pItems[]; +} SModeInfo; + typedef struct SDerivInfo { double prevValue; // previous value TSKEY prevTs; // previous timestamp @@ -4694,21 +4708,99 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) { return pInfo->numOfPoints; } -int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { +bool getModeFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SModeInfo) + MODE_MAX_RESULT_SIZE; + return true; +} + +bool modeFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (!functionSetup(pCtx, pResInfo)) { + return false; + } + + SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + pInfo->numOfPoints = 0; + pInfo->colType = pCtx->resDataInfo.type; + pInfo->colBytes = pCtx->resDataInfo.bytes; + if (pInfo->pHash != NULL) { + taosHashClear(pInfo->pHash); + } else { + pInfo->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + return true; +} + +static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) { + // ignore null elements + if (isNull) { + return; + } + + int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; + SModeItem* pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); + if (pHashItem == NULL) { + int32_t size = sizeof(SModeItem) + pInfo->colBytes; + SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size); + memcpy(pItem->data, data, pInfo->colBytes); + pItem->count += 1; + + taosHashPut(pInfo->pHash, data, hashKeyBytes, (char*)pItem, sizeof(SModeItem*)); + pInfo->numOfPoints++; + } else { + pHashItem->count += 1; + } +} + +int32_t modeFunction(SqlFunctionCtx* pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + + SColumnInfoData* pInputCol = pInput->pData[0]; + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + int32_t startOffset = pCtx->offset; + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + char* data = colDataGetData(pInputCol, i); + doModeAdd(pInfo, data, colDataIsNull_s(pInputCol, i)); + + if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { + taosHashCleanup(pInfo->pHash); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + int32_t currentRow = pBlock->info.rows; - for (int32_t i = 0; i < pResInfo->numOfRes; ++i) { - SUniqueItem* pItem = (SUniqueItem*)(pInfo->pItems + i * (sizeof(SUniqueItem) + pInfo->colBytes)); - colDataAppend(pCol, i, pItem->data, false); - // TODO: handle ts output + int32_t resIndex; + int32_t maxCount = 0; + for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { + SModeItem* pItem = (SModeItem*)(pInfo->pItems + i * (sizeof(SModeItem) + pInfo->colBytes)); + if (pItem->count > maxCount) { + maxCount = pItem->count; + resIndex = i; + } else if (pItem->count == maxCount) { + colDataAppendNULL(pCol, currentRow); + return pResInfo->numOfRes; + } } + SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); + colDataAppend(pCol, currentRow, pResItem->data, false); + return pResInfo->numOfRes; } + bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(STwaInfo); return true; From 24a9d3e51617046fc43e1b8fb7e5b03e29cf7b11 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Tue, 28 Jun 2022 10:05:37 +0800 Subject: [PATCH 04/18] update alter_table.py and common.py --- tests/pytest/util/common.py | 10 +- tests/system-test/1-insert/alter_table.py | 515 +++++++++++----------- 2 files changed, 266 insertions(+), 259 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index b43f87be29..e9db4a5da3 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -637,5 +637,13 @@ class TDCom: column_value_str = ", ".join(str(v) for v in column_value_list) insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});' tsql.execute(insert_sql) - + def getOneRow(self, location, containElm): + res_list = list() + if 0 <= location < tdSql.queryRows: + for row in tdSql.queryResult: + if row[location] == containElm: + res_list.append(row) + return res_list + else: + tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}") tdCom = TDCom() diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index a4e40d1b0b..0f7a830634 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -16,273 +16,272 @@ import string from util.log import * from util.cases import * from util.sql import * +from util import constant +from util.common import * +from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } + self.tag_dict = { + 'ts_tag' : 'timestamp', + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': f'binary({self.binary_length})', + 't13': f'nchar({self.nchar_length})' + } + self.tag_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.tbnum = 1 + self.values_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.column_add_dict = { + 'col_time' : 'timestamp', + 'col_tinyint' : 'tinyint', + 'col_smallint' : 'smallint', + 'col_int' : 'int', + 'col_bigint' : 'bigint', + 'col_untinyint' : 'tinyint unsigned', + 'col_smallint' : 'smallint unsigned', + 'col_int' : 'int unsigned', + 'col_bigint' : 'bigint unsigned', + 'col_bool' : 'bool', + 'col_float' : 'float', + 'col_double' : 'double', + 'col_binary' : f'binary({constant.BINARY_LENGTH_MAX})', + 'col_nchar' : f'nchar({constant.NCAHR_LENGTH_MAX})' - def get_long_name(self, length, mode="mixed"): - """ - generate long name - mode could be numbers/letters/letters_mixed/mixed - """ - if mode == "numbers": - population = string.digits - elif mode == "letters": - population = string.ascii_letters.lower() - elif mode == "letters_mixed": - population = string.ascii_letters.upper() + string.ascii_letters.lower() - else: - population = string.ascii_letters.lower() + string.digits - return "".join(random.choices(population, k=length)) - - def alter_tb_tag_check(self): - tag_tinyint = random.randint(-127,129) - tag_int = random.randint(-2147483648,2147483647) - tag_smallint = random.randint(-32768,32768) - tag_bigint = random.randint(-2147483648,2147483647) - tag_untinyint = random.randint(0,256) - tag_unsmallint = random.randint(0,65536) - tag_unint = random.randint(0,4294967296) - tag_unbigint = random.randint(0,2147483647) - tag_binary = self.get_long_name(length=10, mode="letters") - tag_nchar = self.get_long_name(length=10, mode="letters") - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create stable if not exists {dbname}.{stbname} (col_ts timestamp, c1 int) tags (tag_ts timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20))') - tdSql.execute(f'create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags(now, 1, 2, 3, 4, 5, 6, 7, 8, 9.9, 10.1, True,"abc123","涛思数据")') - tdSql.execute(f'insert into {dbname}.{tbname} values(now, 1)') - tdSql.execute(f'alter table {dbname}.{tbname} set tag tag_ts = 1640966400000') - tdSql.execute(f'alter table {dbname}.{tbname} set tag `t1` = 11') - tdSql.query(f'select * from {dbname}.{stbname}') - tdSql.checkData(0,3,11) - tdSql.execute(f'alter table {dbname}.{tbname} set tag t1 = {tag_tinyint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t2 = {tag_smallint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t3 = {tag_int}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t4 = {tag_bigint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t5 = {tag_untinyint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t6 = {tag_unsmallint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t7 = {tag_unint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t8 = {tag_unbigint}') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t11 = false') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t12 = "{tag_binary}"') - tdSql.execute(f'alter table {dbname}.{tbname} set tag t13 = "{tag_nchar}"') - tdSql.query(f'select * from {dbname}.{stbname}') - # bug TD-15899 - tdSql.checkData(0,2,'2022-01-01 00:00:00.000') - tdSql.checkData(0,3,tag_tinyint) - tdSql.checkData(0,4,tag_smallint) - tdSql.checkData(0,5,tag_int) - tdSql.checkData(0,6,tag_bigint) - tdSql.checkData(0,7,tag_untinyint) - tdSql.checkData(0,8,tag_unsmallint) - tdSql.checkData(0,9,tag_unint) - tdSql.checkData(0,10,tag_unbigint) + } + def alter_check_ntb(self): - tdSql.checkData(0,13,False) - tdSql.checkData(0,14,tag_binary) - tdSql.checkData(0,15,tag_nchar) + tdSql.prepare() + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in self.values_list: + tdSql.execute(f'insert into {self.ntbname} values({i})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter table {self.ntbname} add column {key} {values}') + tdSql.query(f'describe {self.ntbname}') + tdSql.checkRows(len(self.column_dict)+1) + tdSql.query(f'select {key} from {self.ntbname}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter table {self.ntbname} drop column {key}') + tdSql.query(f'describe {self.ntbname}') + tdSql.checkRows(len(self.column_dict)) + tdSql.error(f'select {key} from {self.ntbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.query(f'describe {self.ntbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter table {self.ntbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.ntbname} modify column {key} {v}') + tdSql.query(f'describe {self.ntbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter table {self.ntbname} modify column {key} {v}') + for key,values in self.column_dict.items(): + rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' + tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}') + tdSql.query(f'select {rename_str} from {self.ntbname}') + tdSql.checkRows(1) + + def alter_check_tb(self): + tag_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + tag_smallint = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + tag_int = random.randint(constant.INT_MIN,constant.INT_MAX) + tag_bigint = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + tag_untinyint = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + tag_unsmallint = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + tag_unint = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + tag_unbigint = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + tag_bool = random.randint(0,100)%2 + tag_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + tag_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + tag_binary = tdCom.getLongName(self.binary_length) + tag_nchar = tdCom.getLongName(self.binary_length) + modify_column_dict = { + 'ts1' : 'timestamp', + 'c1': 'tinyint', + 'c2': 'smallint', + 'c3': 'int', + 'c4': 'bigint', + 'c5': 'tinyint unsigned', + 'c6': 'smallint unsigned', + 'c7': 'int unsigned', + 'c8': 'bigint unsigned', + 'c9': 'float', + 'c10': 'double', + 'c11': 'bool', + 'c12': f'binary({self.binary_length})', + 'c13': f'nchar({self.nchar_length})' + } + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for i in range(self.tbnum): + for k,v in modify_column_dict.items(): + tdSql.error(f'alter table {self.stbname}_{i} add column {k} {v}') + for k in self.column_dict.keys(): + tdSql.error(f'alter table {self.stbname}_{i} drop column {k}') + for k,v in self.column_dict.items(): + if 'binary' in v.lower(): + values = [f'binary({self.binary_length+1})', f'binary({self.binary_length-1})'] + for j in values: + tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}') + elif 'nchar' in v.lower(): + values = [f'nchar({self.nchar_length+1})', f'binary({self.nchar_length-1})'] + for j in values: + tdSql.error(f'alter table {self.stbname}_{i} modify {k} {j}') + else: + for values in self.column_dict.values(): + tdSql.error(f'alter table {self.stbname}_{i} modify column {k} {values}') + for k,v in self.tag_dict.items(): + if v.lower() == 'tinyint': + self.tag_check(i,k,tag_tinyint) + elif v.lower() == 'smallint': + self.tag_check(i,k,tag_smallint) + elif v.lower() == 'int': + self.tag_check(i,k,tag_int) + elif v.lower() == 'bigint': + self.tag_check(i,k,tag_bigint) + elif v.lower() == 'tinyint unsigned': + self.tag_check(i,k,tag_untinyint) + elif v.lower() == 'smallint unsigned': + self.tag_check(i,k,tag_unsmallint) + elif v.lower() == 'int unsigned': + self.tag_check(i,k,tag_unint) + elif v.lower() == 'bigint unsigned': + self.tag_check(i,k,tag_unbigint) + elif v.lower() == 'bool': + self.tag_check(i,k,tag_bool) + elif v.lower() == 'float': + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_float}') + tdSql.query(f'select {k} from {self.stbname}_{i}') + if abs(tdSql.queryResult[0][0] - tag_float)/tag_float<=0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + elif v.lower() == 'double': + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = {tag_double}') + tdSql.query(f'select {k} from {self.stbname}_{i}') + if abs(tdSql.queryResult[0][0] - tag_double)/tag_double<=0.0001: + tdSql.checkEqual(tdSql.queryResult[0][0],tdSql.queryResult[0][0]) + else: + tdLog.exit(f'select {k} from {self.stbname}_{i},data check failure') + elif 'binary' in v.lower(): + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_binary}"') + tdSql.query(f'select {k} from {self.stbname}_{i}') + tdSql.checkData(0,0,tag_binary) + elif 'nchar' in v.lower(): + tdSql.execute(f'alter table {self.stbname}_{i} set tag {k} = "{tag_nchar}"') + tdSql.query(f'select {k} from {self.stbname}_{i}') + tdSql.checkData(0,0,tag_nchar) + + def tag_check(self,tb_no,tag,values): + tdSql.execute(f'alter table {self.stbname}_{tb_no} set tag {tag} = {values}') + tdSql.query(f'select {tag} from {self.stbname}_{tb_no}') + tdSql.checkData(0,0,values) + def alter_check_stb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter table {self.stbname} add column {key} {values}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + tdSql.query(f'select {key} from {self.stbname}_{i}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter table {self.stbname} drop column {key}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + tdSql.error(f'select {key} from {self.stbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter table {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter table {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter table {self.stbname} modify column {key} {v}') - # bug TD-16211 insert length more than setting binary and nchar - # error_tag_binary = self.get_long_name(length=21, mode="letters") - # error_tag_nchar = self.get_long_name(length=21, mode="letters") - # tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"') - # tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"') - error_tag_binary = self.get_long_name(length=25, mode="letters") - error_tag_nchar = self.get_long_name(length=25, mode="letters") - tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{error_tag_binary}"') - tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{error_tag_nchar}"') - # bug TD-16210 modify binary to nchar - tdSql.error(f'alter table {dbname}.{tbname} modify tag t12 nchar(10)') - tdSql.execute(f"drop database {dbname}") - def alter_ntb_column_check(self): - ''' - alter ntb column check - ''' - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tbname = self.get_long_name(length=3, mode="letters") - tdLog.info('------------------normal table column check---------------------') - tdLog.info(f'-----------------create normal table {tbname}-------------------') - tdSql.execute(f'create table if not exists {dbname}.{tbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20))') - tdSql.execute(f'insert into {dbname}.{tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - # bug TD-15757 - tdSql.execute(f'alter table {dbname}.{tbname} add column c14 int') - tdSql.query(f'select c14 from {dbname}.{tbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {dbname}.{tbname} add column `c15` int') - tdSql.query(f'select c15 from {dbname}.{tbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {dbname}.{tbname} drop column c14') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter table {dbname}.{tbname} drop column `c15`') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkRows(14) - #! TD-16422 - # tdSql.execute(f'alter table {dbname}.{tbname} add column c16 binary(10)') - # tdSql.query(f'describe {dbname}.{tbname}') - # tdSql.checkRows(15) - # tdSql.checkEqual(tdSql.queryResult[14][2],10) - # tdSql.execute(f'alter table {dbname}.{tbname} drop column c16') - - # tdSql.execute(f'alter table {dbname}.{tbname} add column c16 nchar(10)') - # tdSql.query(f'describe {dbname}.{tbname}') - # tdSql.checkRows(15) - # tdSql.checkEqual(tdSql.queryResult[14][2],10) - # tdSql.execute(f'alter table {dbname}.{tbname} drop column c16') - - - tdSql.execute(f'alter table {dbname}.{tbname} modify column c12 binary(30)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter table {dbname}.{tbname} modify column `c12` binary(35)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 binary(34)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 nchar(10)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c12 int') - tdSql.execute(f'alter table {dbname}.{tbname} modify column c13 nchar(30)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(13,2,30) - tdSql.execute(f'alter table {dbname}.{tbname} modify column `c13` nchar(35)') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(13,2,35) - tdSql.error(f'alter table {dbname}.{tbname} modify column c13 nchar(34)') - tdSql.error(f'alter table {dbname}.{tbname} modify column c13 binary(10)') - tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(1,0,'c21') - # !bug TD-16423 - # tdSql.error(f'select c1 from {dbname}.{tbname}') - # tdSql.query(f'select c21 from {dbname}.{tbname}') - # tdSql.checkData(0,1,1) - tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1') - tdSql.query(f'describe {dbname}.{tbname}') - tdSql.checkData(1,0,'c1') - # !bug TD-16423 - # tdSql.error(f'select c1 from {dbname}.{tbname}') - # tdSql.query(f'select c1 from {dbname}.{tbname}') - # tdSql.checkData(0,1,1) - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int') - tdSql.error(f'alter table {dbname}.{tbname} modify column `c1` double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c9 double') - tdSql.error(f'alter table {dbname}.{tbname} modify column c10 float') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool') - tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)') - tdSql.execute(f'drop database {dbname}') - def alter_stb_column_check(self): - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create table {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'alter table {stbname} add column c14 int') - tdSql.query(f'select c14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {stbname} add column `c15` int') - tdSql.query(f'select c15 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter table {stbname} drop column c14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {stbname} drop column `c15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter table {stbname} modify column c12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter table {stbname} modify column `c12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter table {stbname} modify column `c12` binary(34)') - tdSql.execute(f'alter table {stbname} modify column c13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(13,2,30) - tdSql.error(f'alter table {stbname} modify column c13 nchar(29)') - tdSql.error(f'alter table {stbname} rename column c1 c21') - tdSql.error(f'alter table {stbname} modify column c1 int') - tdSql.error(f'alter table {stbname} modify column c4 int') - tdSql.error(f'alter table {stbname} modify column c8 int') - tdSql.error(f'alter table {stbname} modify column c1 unsigned int') - tdSql.error(f'alter table {stbname} modify column c9 double') - tdSql.error(f'alter table {stbname} modify column c10 float') - tdSql.error(f'alter table {stbname} modify column c11 int') - tdSql.execute(f'drop database {dbname}') - def alter_stb_tag_check(self): - dbname = self.get_long_name(length=10, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - stbname = self.get_long_name(length=3, mode="letters") - tbname = self.get_long_name(length=3, mode="letters") - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create table {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'insert into {tbname} values(now,1)') - - tdSql.execute(f'alter table {stbname} add tag t14 int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter table {stbname} add tag `t15` int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(18) - tdSql.execute(f'alter table {stbname} drop tag t14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter table {stbname} drop tag `t15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter table {stbname} modify tag t12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,30) - tdSql.execute(f'alter table {stbname} modify tag `t12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,35) - tdSql.error(f'alter table {stbname} modify tag `t12` binary(34)') - tdSql.execute(f'alter table {stbname} modify tag t13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(15,2,30) - tdSql.error(f'alter table {stbname} modify tag t13 nchar(29)') - tdSql.execute(f'alter table {stbname} rename tag t1 t21') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t21') - tdSql.execute(f'alter table {stbname} rename tag `t21` t1') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t1') - - for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: - for j in [1,2,3]: - tdSql.error(f'alter table {stbname} modify tag t{j} {i}') - for i in ['int','unsigned int','float','binary(10)','nchar(10)']: - tdSql.error(f'alter table {stbname} modify tag t8 {i}') - tdSql.error(f'alter table {stbname} modify tag t4 int') - tdSql.execute(f'drop database {dbname}') def run(self): - self.alter_tb_tag_check() - self.alter_ntb_column_check() - self.alter_stb_column_check() - self.alter_stb_tag_check() + self.alter_check_ntb() + self.alter_check_tb() + self.alter_check_stb() def stop(self): tdSql.close() From 0c5342730c6d0d072e788abfce39fbef24cf0de8 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 10:34:51 +0800 Subject: [PATCH 05/18] feat: refactor rpc code --- source/libs/transport/inc/transComm.h | 2 +- source/libs/transport/src/transCli.c | 10 +++++----- source/libs/transport/src/transComm.c | 2 +- source/libs/transport/src/transSvr.c | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 963a85922f..55db0b129a 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -229,7 +229,7 @@ typedef struct { SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); void transDestroyAsyncPool(SAsyncPool* pool); -int transSendAsync(SAsyncPool* pool, queue* mq); +int transAsyncSend(SAsyncPool* pool, queue* mq); #define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \ do { \ diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ab08ce82a8..aba2e6957b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -967,7 +967,7 @@ void cliSendQuit(SCliThrd* thrd) { // cli can stop gracefully SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; - transSendAsync(thrd->asyncPool, &msg->q); + transAsyncSend(thrd->asyncPool, &msg->q); } void cliWalkCb(uv_handle_t* handle, void* arg) { if (!uv_is_closing(handle)) { @@ -1138,7 +1138,7 @@ void transReleaseCliHandle(void* handle) { cmsg->msg = tmsg; cmsg->type = Release; - transSendAsync(pThrd->asyncPool, &cmsg->q); + transAsyncSend(pThrd->asyncPool, &cmsg->q); return; } @@ -1171,7 +1171,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra STraceId* trace = &pReq->info.traceId; tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); + ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); return; } @@ -1205,7 +1205,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - transSendAsync(pThrd->asyncPool, &(cliMsg->q)); + transAsyncSend(pThrd->asyncPool, &(cliMsg->q)); tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); @@ -1234,7 +1234,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid); - transSendAsync(thrd->asyncPool, &(cliMsg->q)); + transAsyncSend(thrd->asyncPool, &(cliMsg->q)); } } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index bff7d79bd3..fbe0951a46 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -202,7 +202,7 @@ void transDestroyAsyncPool(SAsyncPool* pool) { taosMemoryFree(pool->asyncs); taosMemoryFree(pool); } -int transSendAsync(SAsyncPool* pool, queue* q) { +int transAsyncSend(SAsyncPool* pool, queue* q) { int idx = pool->index; idx = idx % pool->nAsync; // no need mutex here diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 892d32696e..08363b3c7c 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -999,7 +999,7 @@ void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); - transSendAsync(pThrd->asyncPool, &msg->q); + transAsyncSend(pThrd->asyncPool, &msg->q); } void transCloseServer(void* arg) { @@ -1070,7 +1070,7 @@ void transReleaseSrvHandle(void* handle) { m->type = Release; tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); - transSendAsync(pThrd->asyncPool, &m->q); + transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(refMgt, refId); return; _return1: @@ -1099,7 +1099,7 @@ void transSendResponse(const STransMsg* msg) { STraceId* trace = (STraceId*)&msg->info.traceId; tGTrace("conn %p start to send resp (1/2)", exh->handle); - transSendAsync(pThrd->asyncPool, &m->q); + transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(refMgt, refId); return; _return1: @@ -1128,7 +1128,7 @@ void transRegisterMsg(const STransMsg* msg) { m->type = Register; tTrace("%s conn %p start to register brokenlink callback", transLabel(pThrd->pTransInst), exh->handle); - transSendAsync(pThrd->asyncPool, &m->q); + transAsyncSend(pThrd->asyncPool, &m->q); transReleaseExHandle(refMgt, refId); return; From 4d05053e66f86cc1322d0d53c0a0ff0e511bd925 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 27 Jun 2022 19:41:13 +0800 Subject: [PATCH 06/18] fix(query): add mode function --- source/libs/function/src/builtinsimpl.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ca663ff809..06f24f39e7 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4737,17 +4737,17 @@ static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) { } int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes; - SModeItem* pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); + SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes); if (pHashItem == NULL) { int32_t size = sizeof(SModeItem) + pInfo->colBytes; SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size); memcpy(pItem->data, data, pInfo->colBytes); pItem->count += 1; - taosHashPut(pInfo->pHash, data, hashKeyBytes, (char*)pItem, sizeof(SModeItem*)); + taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*)); pInfo->numOfPoints++; } else { - pHashItem->count += 1; + (*pHashItem)->count += 1; } } @@ -4771,6 +4771,8 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { } } + SET_VAL(pResInfo, 1, 1); + return TSDB_CODE_SUCCESS; } @@ -4789,13 +4791,12 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { maxCount = pItem->count; resIndex = i; } else if (pItem->count == maxCount) { - colDataAppendNULL(pCol, currentRow); - return pResInfo->numOfRes; + resIndex = -1; } } SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); - colDataAppend(pCol, currentRow, pResItem->data, false); + colDataAppend(pCol, currentRow, pResItem->data, (resIndex == -1) ? true : false); return pResInfo->numOfRes; } From f80af20d3b864fb5941b4a3e0f7c243c680115e0 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 28 Jun 2022 10:55:34 +0800 Subject: [PATCH 07/18] fix: add merge parent project and child project optimization --- source/libs/planner/src/planOptimizer.c | 48 ++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 9481c26d7a..eb0f02c92c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1375,19 +1375,57 @@ static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) { } typedef struct SMergeProjectionsContext { - + SProjectLogicNode* pChildProj; + int32_t errCode; } SMergeProjectionsContext; static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { + SMergeProjectionsContext* pCxt = pContext; + SProjectLogicNode* pChildProj = pCxt->pChildProj; + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SNode* pTarget; + FOREACH(pTarget, ((SLogicNode*)pChildProj)->pTargets) { + if (nodesEqualNode(pTarget, *pNode)) { + SNode* pProjection; + FOREACH(pProjection, pChildProj->pProjections) { + if (0 == strcmp(((SColumnNode*)pTarget)->colName, ((SExprNode*)pProjection)->aliasName)) { + SNode* pExpr = nodesCloneNode(pProjection); + if (pExpr == NULL) { + pCxt->errCode = terrno; + return DEAL_RES_ERROR; + } + nodesDestroyNode(*pNode); + *pNode = pExpr; + } + } + } + } + return DEAL_RES_IGNORE_CHILD; + } return DEAL_RES_CONTINUE; } static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pProjectNode) { SProjectLogicNode* pProject = (SProjectLogicNode*)pProjectNode; - SProjectLogicNode* pChild = (SProjectLogicNode*)nodesListGetNode(pProjectNode->pChildren, 0); - SMergeProjectionsContext cxt = {}; - nodesRewriteExprs(pChild->pProjections, mergeProjectionsExpr, &cxt); - return TSDB_CODE_SUCCESS; + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->pChildren, 0); + SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild}; + + nodesRewriteExprs(pProject->pProjections, mergeProjectionsExpr, &cxt); + int32_t code = cxt.errCode; + if (TSDB_CODE_SUCCESS == code) { + if (1 == LIST_LENGTH(pChild->pChildren)) { + SLogicNode* pGrandChild = (SLogicNode*)nodesListGetNode(pChild->pChildren, 0); + code = replaceLogicNode(pLogicSubplan, pChild, pGrandChild); + } else { // no grand child + NODES_CLEAR_LIST(pProjectNode->pChildren); + } + } + + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pChild->pChildren); + } + nodesDestroyNode((SNode*)pChild); + return code; } static int32_t mergeProjectsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { From 97bddc916edd99dd6f6bf0d7f389e888de88b81e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 28 Jun 2022 11:00:45 +0800 Subject: [PATCH 08/18] test: sim test timeout --- tests/script/test-all.bat | 2 +- tests/script/wtest.bat | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat index 81c1f21c04..1c698449c6 100644 --- a/tests/script/test-all.bat +++ b/tests/script/test-all.bat @@ -19,7 +19,7 @@ for /F "usebackq tokens=*" %%i in (!caseFile!) do ( call :GetTimeSeconds !time! set time1=!_timeTemp! echo Start at !time! - call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt || set /a errorlevel=8 if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) ) ) diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 79daf01295..25c344577a 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -56,8 +56,14 @@ echo charset UTF-8 >> %TAOS_CFG% set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" +set FILE_NAME=%FILE_NAME:/=\% + +start cmd /k "timeout /t 600 && taskkill /f /im tsim.exe & exit /b" rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% +set result=false +%TSIM% -c %CFG_DIR% -f %FILE_NAME% && set result=true -%TSIM% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file +tasklist | grep timeout && taskkill /f /im timeout.exe +if "%result%" == "true" ( exit /b ) else ( exit /b 8 ) \ No newline at end of file From 4c410bd5eceda845ed6850d986cbe516f7a4cc80 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 28 Jun 2022 11:33:14 +0800 Subject: [PATCH 09/18] refactor(sync): refactor snapshot code --- source/libs/sync/inc/syncSnapshot.h | 4 +- source/libs/sync/src/syncAppendEntriesReply.c | 28 +- source/libs/sync/src/syncSnapshot.c | 247 ++++++++++++------ 3 files changed, 168 insertions(+), 111 deletions(-) diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 55e0755a89..14688f8912 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -57,7 +57,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader); -void snapshotSenderStop(SSyncSnapshotSender *pSender); +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); @@ -82,7 +82,7 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index b7122b9c52..10f2745651 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -240,7 +240,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); ASSERT(pSender != NULL); - SSnapshot snapshot; + SSnapshot snapshot = {.data = NULL, + .lastApplyIndex = SYNC_INDEX_INVALID, + .lastApplyTerm = 0, + .lastConfigIndex = SYNC_INDEX_INVALID}; void* pReader = NULL; ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot, NULL, &pReader); if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN && nextIndex <= snapshot.lastApplyIndex + 1 && @@ -249,10 +252,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries ASSERT(pReader != NULL); snapshotSenderStart(pSender, snapshot, pReader); - char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); - syncNodeEventLog(ths, eventLog); - taosMemoryFree(eventLog); - } else { // no snapshot if (pReader != NULL) { @@ -260,23 +259,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } } - /* - bool hasSnapshot = syncNodeHasSnapshot(ths); - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - - // start sending snapshot first time - // start here, stop by receiver - if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && - pMsg->privateTerm < pSender->privateTerm) { - snapshotSenderStart(pSender); - - char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); - syncNodeEventLog(ths, eventLog); - taosMemoryFree(eventLog); - } - */ - SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; // update nextIndex to sentryIndex @@ -300,5 +282,5 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrLog2("recv sync-append-entries-reply, after pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv sync-append-entries-reply, after pMatchIndex:", ths->pMatchIndex); - return ret; + return 0; } \ No newline at end of file diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 39e972c4c4..7b15be2d14 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -24,6 +24,7 @@ //---------------------------------- static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg); +static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg); //---------------------------------- SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex) { @@ -50,7 +51,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot)); pSender->finish = false; } else { - sError("vgId:%d cannot create snapshot sender", pSyncNode->vgId); + sError("vgId:%d, cannot create snapshot sender", pSyncNode->vgId); } return pSender; @@ -127,7 +128,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void syncEntryDestory(pEntry); } else { if (pSender->snapshot.lastConfigIndex == pSender->pSyncNode->pRaftCfg->lastConfigIndex) { - sTrace("vgId:%d sync sender get cfg from local", pSender->pSyncNode->vgId); + sTrace("vgId:%d, sync sender get cfg from local", pSender->pSyncNode->vgId); pSender->lastConfig = pSender->pSyncNode->pRaftCfg->cfg; getLastConfig = true; } @@ -176,13 +177,13 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void // event log do { - char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send"); + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); syncNodeEventLog(pSender->pSyncNode, eventLog); taosMemoryFree(eventLog); } while (0); } -void snapshotSenderStop(SSyncSnapshotSender *pSender) { +void snapshotSenderStop(SSyncSnapshotSender *pSender, bool finish) { // close reader if (pSender->pReader != NULL) { int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader); @@ -199,6 +200,14 @@ void snapshotSenderStop(SSyncSnapshotSender *pSender) { // update flag pSender->start = false; + pSender->finish = finish; + + // event log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender stop"); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } // when sender receive ack, call this function to send msg from seq @@ -386,7 +395,7 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID; } else { - sError("vgId:%d cannot create snapshot receiver", pSyncNode->vgId); + sError("vgId:%d, cannot create snapshot receiver", pSyncNode->vgId); } return pReceiver; @@ -409,9 +418,9 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } -// static do start +// static do start by privateTerm, pBeginMsg // receive first snapshot data -// privateTerm, pBeginMsg +// write first block data static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { // update state @@ -419,6 +428,7 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm p pReceiver->privateTerm = privateTerm; pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN; pReceiver->fromId = pBeginMsg->srcId; + pReceiver->start = true; // update snapshot pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex; @@ -429,8 +439,16 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm p ASSERT(pReceiver->pWriter == NULL); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter)); ASSERT(ret == 0); + + // event log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver start"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } +// force stop static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { @@ -441,33 +459,35 @@ static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { } pReceiver->start = false; + + // event log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver force stop"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver // if already start, force close, start again void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) { if (!snapshotReceiverIsStart(pReceiver)) { - // start + // first start snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); - pReceiver->start = true; } else { // already start - sInfo("snapshot recv, receiver already start"); + sInfo("vgId:%d, snapshot recv, receiver already start", pReceiver->pSyncNode->vgId); // force close, abandon incomplete data - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); - ASSERT(ret == 0); - pReceiver->pWriter = NULL; + snapshotReceiverForceStop(pReceiver); // start again snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg); - pReceiver->start = true; } } -void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { +void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { if (pReceiver->pWriter != NULL) { int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); @@ -477,8 +497,69 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) { pReceiver->start = false; - if (apply) { - // ++(pReceiver->privateTerm); + // event log + do { + SSnapshot snapshot; + pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver stop"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); +} + +static void snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { + ASSERT(pMsg->seq == SYNC_SNAPSHOT_SEQ_END); + + if (pReceiver->pWriter != NULL) { + int32_t code = 0; + if (pMsg->dataLen > 0) { + code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, + pMsg->dataLen); + ASSERT(code == 0); + } + + code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true); + ASSERT(code == 0); + pReceiver->pWriter = NULL; + } + + pReceiver->ack = SYNC_SNAPSHOT_SEQ_END; + + // update commit index + if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) { + pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; + } + + // reset wal + pReceiver->pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pReceiver->pSyncNode->pLogStore, pMsg->lastIndex); + + // event log + do { + SSnapshot snapshot; + pReceiver->pSyncNode->pFsm->FpGetSnapshotInfo(pReceiver->pSyncNode->pFsm, &snapshot); + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver got last data, finish, apply snapshot"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); +} + +static void snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) { + ASSERT(pMsg->seq == pReceiver->ack + 1); + + if (pReceiver->pWriter != NULL) { + if (pMsg->dataLen > 0) { + int32_t code = pReceiver->pSyncNode->pFsm->FpSnapshotDoWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, + pMsg->data, pMsg->dataLen); + ASSERT(code == 0); + } + pReceiver->ack = pMsg->seq; + + // event log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver receiving"); + syncNodeEventLog(pReceiver->pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } @@ -560,33 +641,20 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // get receiver SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver; bool needRsp = false; - int32_t writeCode = 0; // state, term, seq/ack if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_BEGIN) { - // begin + // begin, no data snapshotReceiverStart(pReceiver, pMsg->privateTerm, pMsg); - pReceiver->ack = pMsg->seq; needRsp = true; - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver begin"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_END) { // end, finish FSM - writeCode = pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); - ASSERT(writeCode == 0); - - pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); - if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) { - pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; - } - - // pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); - pSyncNode->pLogStore->syncLogRestoreFromSnapshot(pSyncNode->pLogStore, pMsg->lastIndex); + snapshotReceiverFinish(pReceiver, pMsg); + snapshotReceiverStop(pReceiver); + needRsp = true; // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { @@ -601,89 +669,74 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { syncNodeDoConfigChange(pSyncNode, &newSyncCfg, pMsg->lastConfigIndex); } - SSnapshot snapshot; - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver finish, apply snapshot"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - - pReceiver->pWriter = NULL; - snapshotReceiverStop(pReceiver, true); - pReceiver->ack = pMsg->seq; - needRsp = true; - - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver stop"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } else if (pMsg->seq == SYNC_SNAPSHOT_SEQ_FORCE_CLOSE) { - pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, false); - snapshotReceiverStop(pReceiver, false); + // force close + snapshotReceiverForceStop(pReceiver); needRsp = false; - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver force close"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } else if (pMsg->seq > SYNC_SNAPSHOT_SEQ_BEGIN && pMsg->seq < SYNC_SNAPSHOT_SEQ_END) { // transfering if (pMsg->seq == pReceiver->ack + 1) { - writeCode = - pSyncNode->pFsm->FpSnapshotDoWrite(pSyncNode->pFsm, pReceiver->pWriter, pMsg->data, pMsg->dataLen); - ASSERT(writeCode == 0); - pReceiver->ack = pMsg->seq; + snapshotReceiverGotData(pReceiver, pMsg); } needRsp = true; - do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver receiving"); - syncNodeEventLog(pSyncNode, eventLog); - taosMemoryFree(eventLog); - } while (0); - } else { ASSERT(0); } + // send ack if (needRsp) { + // build msg SyncSnapshotRsp *pRspMsg = syncSnapshotRspBuild(pSyncNode->vgId); pRspMsg->srcId = pSyncNode->myRaftId; pRspMsg->destId = pMsg->srcId; pRspMsg->term = pSyncNode->pRaftStore->currentTerm; pRspMsg->lastIndex = pMsg->lastIndex; pRspMsg->lastTerm = pMsg->lastTerm; - pRspMsg->ack = pReceiver->ack; - pRspMsg->code = writeCode; - pRspMsg->privateTerm = pReceiver->privateTerm; + pRspMsg->ack = pReceiver->ack; // receiver maybe already closed + pRspMsg->code = 0; + pRspMsg->privateTerm = pReceiver->privateTerm; // receiver maybe already closed + // send msg SRpcMsg rpcMsg; syncSnapshotRsp2RpcMsg(pRspMsg, &rpcMsg); syncNodeSendMsgById(&(pRspMsg->destId), pSyncNode, &rpcMsg); - syncSnapshotRspDestroy(pRspMsg); } + } else { + // error log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver term not equal"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } } else { - syncNodeLog2("syncNodeOnSnapshotSendCb not follower", pSyncNode); + // error log + do { + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver not follower"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); } return 0; } +static void snapshotSenderUpdateProgress(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + ASSERT(pMsg->ack == pSender->seq); + pSender->ack = pMsg->ack; + ++(pSender->seq); +} + // sender receives ack, set seq = ack + 1, send msg from seq // if ack == SYNC_SNAPSHOT_SEQ_END, stop sender int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { // if already drop replica, do not process if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - sInfo("recv SyncSnapshotRsp maybe replica already dropped"); - return 0; + sError("vgId:%d, recv sync-snapshot-rsp, maybe replica already dropped", pSyncNode->vgId); + return -1; } // get sender @@ -695,27 +748,49 @@ int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { if (pMsg->term == pSyncNode->pRaftStore->currentTerm) { // receiver ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { - pSender->finish = true; - snapshotSenderStop(pSender); + snapshotSenderStop(pSender, true); return 0; } // send next msg if (pMsg->ack == pSender->seq) { // update sender ack - pSender->ack = pMsg->ack; - (pSender->seq)++; + snapshotSenderUpdateProgress(pSender, pMsg); snapshotSend(pSender); } else if (pMsg->ack == pSender->seq - 1) { snapshotReSend(pSender); } else { - ASSERT(0); + do { + char logBuf[64]; + snprintf(logBuf, sizeof(logBuf), "error ack, recv ack:%d, my seq:%d", pMsg->ack, pSender->seq); + char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + + return -1; } + } else { + // error log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender term not equal"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + + return -1; } } else { - syncNodeLog2("syncNodeOnSnapshotRspCb not leader", pSyncNode); + // error log + do { + char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender not leader"); + syncNodeErrorLog(pSyncNode, eventLog); + taosMemoryFree(eventLog); + } while (0); + + return -1; } return 0; From acb7f040aa04f9cd795dc4e802f9c9245fb32e50 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 28 Jun 2022 11:48:10 +0800 Subject: [PATCH 10/18] fix(stream): taosd crash when partition by tbname --- source/libs/executor/src/executorimpl.c | 3 +-- tests/script/jenkins/basic.txt | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 17e5482b60..ad0d5c1447 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -273,8 +273,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR } // 1. close current opened time window - if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && - pResult->offset != pResultRowInfo->cur.offset))) { + if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) { SResultRowPosition pos = pResultRowInfo->cur; SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); releaseBufPage(pResultBuf, pPage); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d3a6a0d102..ab5117ced8 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -85,6 +85,7 @@ ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/distributeInterval0.sim +# ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim # ./test.sh -f tsim/stream/distributesession0.sim # ./test.sh -f tsim/stream/session0.sim ./test.sh -f tsim/stream/session1.sim From 20533da6d09d267246838e5bc1000d48f1be8a3e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 28 Jun 2022 11:52:46 +0800 Subject: [PATCH 11/18] test: sim test timeout --- tests/script/wtest.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 25c344577a..e3bbff9db5 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -58,7 +58,7 @@ set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" set FILE_NAME=%FILE_NAME:/=\% -start cmd /k "timeout /t 600 && taskkill /f /im tsim.exe & exit /b" +start cmd /k "timeout /t 600 /NOBREAK && taskkill /f /im tsim.exe & exit /b" rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% From 9540d1eac9df7d577ced628bb11b3ea0860ff1b1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 28 Jun 2022 11:54:16 +0800 Subject: [PATCH 12/18] feat: udf can be compiled outside TDengine --- include/libs/function/taosudf.h | 266 +++++++++++++++++++++++++++++++ include/libs/function/tudf.h | 228 +------------------------- source/libs/function/test/udf1.c | 6 +- source/libs/function/test/udf2.c | 8 +- 4 files changed, 276 insertions(+), 232 deletions(-) create mode 100644 include/libs/function/taosudf.h diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h new file mode 100644 index 0000000000..5e84b87a81 --- /dev/null +++ b/include/libs/function/taosudf.h @@ -0,0 +1,266 @@ +/* +* Copyright (c) 2019 TAOS Data, Inc. +* +* This program is free software: you can use, redistribute, and/or modify +* it under the terms of the GNU Affero General Public License, version 3 +* or later ("AGPL"), as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +*/ + +#ifndef TDENGINE_TAOSUDF_H +#define TDENGINE_TAOSUDF_H + +#include +#include +#include +#include + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__GNUC__) +#define FORCE_INLINE inline __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif +typedef struct SUdfColumnMeta { + int16_t type; + int32_t bytes; + uint8_t precision; + uint8_t scale; +} SUdfColumnMeta; + +typedef struct SUdfColumnData { + int32_t numOfRows; + int32_t rowsAlloc; + union { + struct { + int32_t nullBitmapLen; + char *nullBitmap; + int32_t dataLen; + char *data; + } fixLenCol; + + struct { + int32_t varOffsetsLen; + int32_t *varOffsets; + int32_t payloadLen; + char *payload; + int32_t payloadAllocLen; + } varLenCol; + }; +} SUdfColumnData; + + +typedef struct SUdfColumn { + SUdfColumnMeta colMeta; + bool hasNull; + SUdfColumnData colData; +} SUdfColumn; + +typedef struct SUdfDataBlock { + int32_t numOfRows; + int32_t numOfCols; + SUdfColumn **udfCols; +} SUdfDataBlock; + +typedef struct SUdfInterBuf { + int32_t bufLen; + char* buf; + int8_t numOfResult; //zero or one +} SUdfInterBuf; +typedef void *UdfcFuncHandle; + +// dynamic lib init and destroy +typedef int32_t (*TUdfInitFunc)(); +typedef int32_t (*TUdfDestroyFunc)(); + +#define UDF_MEMORY_EXP_GROWTH 1.5 +#define NBIT (3u) +#define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) +#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) +#define BitmapLen(_n) (((_n) + ((1 << NBIT) - 1)) >> NBIT) + +#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1) +#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row)))) +#define udfColDataSetNull_f(pColumn, row) \ + do { \ + BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \ + } while (0) + +#define udfColDataSetNotNull_f(pColumn, r_) \ + do { \ + BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_))); \ + } while (0) +#define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1) + +typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE) +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) +#define IS_VAR_DATA_TYPE(t) \ + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) +#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) + + +static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) { + if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { + return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row]; + } else { + return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row; + } +} + +static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) { + if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { + if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) { + if (udfColDataIsNull_var(pColumn, row)) { + return true; + } + char* data = udfColDataGetData(pColumn, row); + return (*data == TSDB_DATA_TYPE_NULL); + } else { + return udfColDataIsNull_var(pColumn, row); + } + } else { + return udfColDataIsNull_f(pColumn, row); + } +} + +static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + + if (newCapacity== 0 || newCapacity <= data->rowsAlloc) { + return TSDB_CODE_SUCCESS; + } + + int allocCapacity = (data->rowsAlloc< 8) ? 8 : data->rowsAlloc; + while (allocCapacity < newCapacity) { + allocCapacity *= UDF_MEMORY_EXP_GROWTH; + } + + if (IS_VAR_DATA_TYPE(meta->type)) { + char* tmp = (char*)realloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->varLenCol.varOffsets = (int32_t*)tmp; + data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; + // for payload, add data in udfColDataAppend + } else { + char* tmp = (char*)realloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->fixLenCol.nullBitmap = tmp; + data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); + if (meta->type == TSDB_DATA_TYPE_NULL) { + return TSDB_CODE_SUCCESS; + } + + tmp = (char*)realloc(data->fixLenCol.data, allocCapacity* meta->bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->fixLenCol.data = tmp; + data->fixLenCol.dataLen = allocCapacity* meta->bytes; + } + + data->rowsAlloc = allocCapacity; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) { + udfColEnsureCapacity(pColumn, row+1); + if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { + udfColDataSetNull_var(pColumn, row); + } else { + udfColDataSetNull_f(pColumn, row); + } + pColumn->hasNull = true; +} + +static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + udfColEnsureCapacity(pColumn, currentRow+1); + bool isVarCol = IS_VAR_DATA_TYPE(meta->type); + if (isNull) { + udfColDataSetNull(pColumn, currentRow); + } else { + if (!isVarCol) { + udfColDataSetNotNull_f(pColumn, currentRow); + memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); + } else { + int32_t dataLen = varDataTLen(pData); + if (meta->type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + sizeof(char)); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = sizeof(int64_t); + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = sizeof(char); + } + dataLen += sizeof(char); + } + + if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { + uint32_t newSize = data->varLenCol.payloadAllocLen; + if (newSize <= 1) { + newSize = 8; + } + + while (newSize < data->varLenCol.payloadLen + dataLen) { + newSize = newSize * UDF_MEMORY_EXP_GROWTH; + } + + char *buf = (char*)realloc(data->varLenCol.payload, newSize); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->varLenCol.payload = buf; + data->varLenCol.payloadAllocLen = newSize; + } + + uint32_t len = data->varLenCol.payloadLen; + data->varLenCol.varOffsets[currentRow] = len; + + memcpy(data->varLenCol.payload + len, pData, dataLen); + data->varLenCol.payloadLen += dataLen; + } + } + data->numOfRows = (currentRow + 1 > data->numOfRows) ? (currentRow+1) : data->numOfRows; + return 0; +} + +typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); + +typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); +typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf); +typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TAOSUDF_H diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 28b1fbe8ce..b4c05fea87 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -16,6 +16,13 @@ #ifndef TDENGINE_TUDF_H #define TDENGINE_TUDF_H +#undef malloc +#define malloc malloc +#undef free +#define free free +#undef realloc +#define alloc alloc +#include #include #include @@ -36,56 +43,6 @@ extern "C" { #endif #define UDF_DNODE_ID_ENV_NAME "DNODE_ID" -//====================================================================================== -//begin API to taosd and qworker - -typedef struct SUdfColumnMeta { - int16_t type; - int32_t bytes; - uint8_t precision; - uint8_t scale; -} SUdfColumnMeta; - -typedef struct SUdfColumnData { - int32_t numOfRows; - int32_t rowsAlloc; - union { - struct { - int32_t nullBitmapLen; - char *nullBitmap; - int32_t dataLen; - char *data; - } fixLenCol; - - struct { - int32_t varOffsetsLen; - int32_t *varOffsets; - int32_t payloadLen; - char *payload; - int32_t payloadAllocLen; - } varLenCol; - }; -} SUdfColumnData; - - -typedef struct SUdfColumn { - SUdfColumnMeta colMeta; - bool hasNull; - SUdfColumnData colData; -} SUdfColumn; - -typedef struct SUdfDataBlock { - int32_t numOfRows; - int32_t numOfCols; - SUdfColumn **udfCols; -} SUdfDataBlock; - -typedef struct SUdfInterBuf { - int32_t bufLen; - char* buf; - int8_t numOfResult; //zero or one -} SUdfInterBuf; -typedef void *UdfcFuncHandle; //low level APIs /** @@ -127,177 +84,6 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); int32_t cleanUpUdfs(); -// end API to taosd and qworker -//============================================================================================================================= -// begin API to UDF writer. - -// dynamic lib init and destroy -typedef int32_t (*TUdfInitFunc)(); -typedef int32_t (*TUdfDestroyFunc)(); - -//TODO: add API to check function arguments type, number etc. - -#define UDF_MEMORY_EXP_GROWTH 1.5 - -#define udfColDataIsNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] == -1) -#define udfColDataIsNull_f(pColumn, row) ((BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) & (1u << (7u - BitPos(row)))) == (1u << (7u - BitPos(row)))) -#define udfColDataSetNull_f(pColumn, row) \ - do { \ - BMCharPos(pColumn->colData.fixLenCol.nullBitmap, row) |= (1u << (7u - BitPos(row))); \ - } while (0) - -#define udfColDataSetNotNull_f(pColumn, r_) \ - do { \ - BMCharPos(pColumn->colData.fixLenCol.nullBitmap, r_) &= ~(1u << (7u - BitPos(r_))); \ - } while (0) -#define udfColDataSetNull_var(pColumn, row) ((pColumn->colData.varLenCol.varOffsets)[row] = -1) - - -static FORCE_INLINE char* udfColDataGetData(const SUdfColumn* pColumn, int32_t row) { - if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { - return pColumn->colData.varLenCol.payload + pColumn->colData.varLenCol.varOffsets[row]; - } else { - return pColumn->colData.fixLenCol.data + pColumn->colMeta.bytes * row; - } -} - -static FORCE_INLINE bool udfColDataIsNull(const SUdfColumn* pColumn, int32_t row) { - if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { - if (pColumn->colMeta.type == TSDB_DATA_TYPE_JSON) { - if (udfColDataIsNull_var(pColumn, row)) { - return true; - } - char* data = udfColDataGetData(pColumn, row); - return (*data == TSDB_DATA_TYPE_NULL); - } else { - return udfColDataIsNull_var(pColumn, row); - } - } else { - return udfColDataIsNull_f(pColumn, row); - } -} - -static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) { - SUdfColumnMeta *meta = &pColumn->colMeta; - SUdfColumnData *data = &pColumn->colData; - - if (newCapacity== 0 || newCapacity <= data->rowsAlloc) { - return TSDB_CODE_SUCCESS; - } - - int allocCapacity = TMAX(data->rowsAlloc, 8); - while (allocCapacity < newCapacity) { - allocCapacity *= UDF_MEMORY_EXP_GROWTH; - } - - if (IS_VAR_DATA_TYPE(meta->type)) { - char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - data->varLenCol.varOffsets = (int32_t*)tmp; - data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; - // for payload, add data in udfColDataAppend - } else { - char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - data->fixLenCol.nullBitmap = tmp; - data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); - if (meta->type == TSDB_DATA_TYPE_NULL) { - return TSDB_CODE_SUCCESS; - } - - tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - data->fixLenCol.data = tmp; - data->fixLenCol.dataLen = allocCapacity* meta->bytes; - } - - data->rowsAlloc = allocCapacity; - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) { - udfColEnsureCapacity(pColumn, row+1); - if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) { - udfColDataSetNull_var(pColumn, row); - } else { - udfColDataSetNull_f(pColumn, row); - } - pColumn->hasNull = true; -} - -static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { - SUdfColumnMeta *meta = &pColumn->colMeta; - SUdfColumnData *data = &pColumn->colData; - udfColEnsureCapacity(pColumn, currentRow+1); - bool isVarCol = IS_VAR_DATA_TYPE(meta->type); - if (isNull) { - udfColDataSetNull(pColumn, currentRow); - } else { - if (!isVarCol) { - colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow); - memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); - } else { - int32_t dataLen = varDataTLen(pData); - if (meta->type == TSDB_DATA_TYPE_JSON) { - if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = 0; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = LONG_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { - dataLen = CHAR_BYTES; - } - dataLen += CHAR_BYTES; - } - - if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { - uint32_t newSize = data->varLenCol.payloadAllocLen; - if (newSize <= 1) { - newSize = 8; - } - - while (newSize < data->varLenCol.payloadLen + dataLen) { - newSize = newSize * UDF_MEMORY_EXP_GROWTH; - } - - char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize); - if (buf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - data->varLenCol.payload = buf; - data->varLenCol.payloadAllocLen = newSize; - } - - uint32_t len = data->varLenCol.payloadLen; - data->varLenCol.varOffsets[currentRow] = len; - - memcpy(data->varLenCol.payload + len, pData, dataLen); - data->varLenCol.payloadLen += dataLen; - } - } - data->numOfRows = TMAX(currentRow + 1, data->numOfRows); - return 0; -} - -typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); - -typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf); -typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf); -typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData); - - -// end API to UDF writer -//======================================================================================================================= #ifdef __cplusplus } diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index 9443d5cb94..dfbae357ef 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -2,12 +2,8 @@ #include #include -#include "tudf.h" +#include "taosudf.h" -#undef malloc -#define malloc malloc -#undef free -#define free free DLL_EXPORT int32_t udf1_init() { return 0; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 1c270f5cf4..975832209e 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -1,13 +1,9 @@ #include #include #include +#include -#include "tudf.h" - -#undef malloc -#define malloc malloc -#undef free -#define free free +#include "taosudf.h" DLL_EXPORT int32_t udf2_init() { return 0; From 4110c0049739804159c6c66fff8b10c67598dd58 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 28 Jun 2022 11:58:39 +0800 Subject: [PATCH 13/18] fix: initialize errCode of SMergeProjectsContext --- source/libs/planner/src/planOptimizer.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index eb0f02c92c..638e3f5604 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1405,19 +1405,19 @@ static EDealRes mergeProjectionsExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pProjectNode) { - SProjectLogicNode* pProject = (SProjectLogicNode*)pProjectNode; - SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->pChildren, 0); - SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild}; +static int32_t mergeProjectsOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSelfNode) { + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSelfNode->pChildren, 0); + SMergeProjectionsContext cxt = {.pChildProj = (SProjectLogicNode*)pChild, .errCode = TSDB_CODE_SUCCESS}; - nodesRewriteExprs(pProject->pProjections, mergeProjectionsExpr, &cxt); + nodesRewriteExprs(((SProjectLogicNode*)pSelfNode)->pProjections, mergeProjectionsExpr, &cxt); int32_t code = cxt.errCode; + if (TSDB_CODE_SUCCESS == code) { if (1 == LIST_LENGTH(pChild->pChildren)) { SLogicNode* pGrandChild = (SLogicNode*)nodesListGetNode(pChild->pChildren, 0); code = replaceLogicNode(pLogicSubplan, pChild, pGrandChild); } else { // no grand child - NODES_CLEAR_LIST(pProjectNode->pChildren); + NODES_CLEAR_LIST(pSelfNode->pChildren); } } From 5be089b888fb35efc4536d8abdcd26a5a37feffc Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 28 Jun 2022 14:11:44 +0800 Subject: [PATCH 14/18] refactor: rsma restore --- source/dnode/vnode/src/inc/sma.h | 23 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/sma/smaOpen.c | 8 +- source/dnode/vnode/src/sma/smaRollup.c | 631 +++++++++++++------------ source/dnode/vnode/src/sma/smaUtil.c | 42 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 +- 6 files changed, 357 insertions(+), 353 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 4b42ab5263..63ac6397bd 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -28,7 +28,8 @@ extern "C" { #define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) #define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +// #define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define smaDebug(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on @@ -205,16 +206,16 @@ struct STFile { uint8_t state; }; -#define TD_FILE_F(tf) (&((tf)->f)) -#define TD_FILE_PFILE(tf) ((tf)->pFile) -#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) -#define TD_FILE_FULL_NAME(tf) (TD_FILE_F(tf)->aname) -#define TD_FILE_REL_NAME(tf) (TD_FILE_F(tf)->rname) -#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) -#define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf)) -#define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL) -#define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s)) -#define TD_FILE_DID(tf) (TD_FILE_F(tf)->did) +#define TD_TFILE_F(tf) (&((tf)->f)) +#define TD_TFILE_PFILE(tf) ((tf)->pFile) +#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) +#define TD_TFILE_FULL_NAME(tf) (TD_TFILE_F(tf)->aname) +#define TD_TFILE_REL_NAME(tf) (TD_TFILE_F(tf)->rname) +#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) +#define TD_TFILE_CLOSED(tf) (!TD_TFILE_OPENED(tf)) +#define TD_TFILE_SET_CLOSED(f) (TD_TFILE_PFILE(f) = NULL) +#define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TD_TFILE_DID(tf) (TD_TFILE_F(tf)->did) int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 68ed6dde51..baead763ad 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -64,6 +64,7 @@ typedef struct STsdbSnapshotReader STsdbSnapshotReader; #define VNODE_TQ_DIR "tq" #define VNODE_WAL_DIR "wal" #define VNODE_TSMA_DIR "tsma" +#define VNODE_RSMA_DIR "rsma" #define VNODE_RSMA0_DIR "tsdb" #define VNODE_RSMA1_DIR "rsma1" #define VNODE_RSMA2_DIR "rsma2" @@ -161,7 +162,6 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool // sma int32_t smaOpen(SVnode* pVnode); -int32_t smaClose(SSma* pSma); int32_t smaCloseEnv(SSma* pSma); int32_t smaCloseEx(SSma* pSma); diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 88ed7426f7..641b8c7934 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -123,7 +123,7 @@ int32_t smaOpen(SVnode *pVnode) { } // restore the rsma -#if 0 +#if 1 if (rsmaRestore(pSma) < 0) { goto _err; } @@ -154,12 +154,6 @@ int32_t smaCloseEx(SSma *pSma) { return 0; } -int32_t smaClose(SSma *pSma) { - smaCloseEnv(pSma); - smaCloseEx(pSma); - return 0; -} - /** * @brief rsma env restore * diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ed5b6f4055..8423654eee 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,13 +15,14 @@ #include "sma.h" -#define RSMA_QTASKINFO_PERSIST_MS 7200000 +#define RSMA_QTASKINFO_PERSIST_MS 5000 // 7200000 #define RSMA_QTASKINFO_BUFSIZE 32768 +#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; -typedef struct SRSmaQTaskFIter SRSmaQTaskFIter; +typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); @@ -32,11 +33,11 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType static void tdRSmaFetchTrigger(void *param, void *tmrId); static void tdRSmaPersistTrigger(void *param, void *tmrId); static void *tdRSmaPersistExec(void *param); -static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName); +static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName); -static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile); -static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish); -static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd); +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); +static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter); static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem); struct SRSmaInfoItem { @@ -63,22 +64,23 @@ struct SRSmaQTaskInfoItem { void *qTaskInfo; }; -struct SRSmaQTaskFIter { +struct SRSmaQTaskInfoIter { STFile *pTFile; int64_t offset; int64_t fsize; int32_t nBytes; int32_t nAlloc; - char *buf; + char *pBuf; // ------------ + char *qBuf; // for iterator int32_t nBufPos; }; static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { - return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t); + return lenWithHead - RSMA_QTASKINFO_HEAD_LEN; } -static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); } +static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); } static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC @@ -294,7 +296,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { - ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally + ASSERT(0); // TODO: free original pRSmaInfo if exists abnormally smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; } @@ -338,10 +340,10 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; - } else { - smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid); } + smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid); + // start the persist timer if (TASK_TRIGGER_STAT_INIT == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) { @@ -356,10 +358,9 @@ _err: } /** - * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam currently * - * @param pTsdb - * @param pMeta + * @param pVnode * @param pReq * @return int32_t */ @@ -695,8 +696,257 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } -static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) { - tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); +int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + + // step 1: iterate all stables to restore the rsma env + SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); + if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { + taosArrayDestroy(suidList); + smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr()); + return TSDB_CODE_FAILED; + } + + int32_t arrSize = taosArrayGetSize(suidList); + if (arrSize == 0) { + taosArrayDestroy(suidList); + smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode)); + return TSDB_CODE_SUCCESS; + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + for (int32_t i = 0; i < arrSize; ++i) { + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + smaDebug("vgId:%d, rsma restore, suid[%d] is %" PRIi64, TD_VID(pVnode), i, suid); + if (metaGetTableEntryByUid(&mr, suid) < 0) { + smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + SRSmaParam *param = &mr.me.stbEntry.rsmaParam; + for (int i = 0; i < TSDB_RETENTION_L2; ++i) { + smaDebug("vgId:%d, rsma restore, table:%" PRIi64 " level:%d, maxdelay:%" PRIi64 " watermark:%" PRIi64 + " qmsgLen:%" PRIi32, + TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]); + } + if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { + smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); + goto _err; + } + smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid); + } + } + + // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore + STFile tFile = {0}; + char qTaskInfoFName[TSDB_FILENAME_LEN]; + + tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName); + if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { + goto _err; + } + if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { + goto _err; + } + SRSmaQTaskInfoIter fIter = {0}; + if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { + goto _err; + } + SRSmaQTaskInfoItem infoItem = {0}; + if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) { + tdRSmaQTaskInfoIterDestroy(&fIter); + goto _err; + } + + tdRSmaQTaskInfoIterDestroy(&fIter); + metaReaderClear(&mr); + taosArrayDestroy(suidList); + return TSDB_CODE_SUCCESS; +_err: + metaReaderClear(&mr); + taosArrayDestroy(suidList); + smaError("failed to restore rsma task since %s", terrstr()); + return TSDB_CODE_FAILED; +} + +static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); + SRSmaInfo *pRSmaInfo = NULL; + void *qTaskInfo = NULL; + + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pItem->suid, sizeof(pItem->suid)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (pItem->type == 1) { + qTaskInfo = pRSmaInfo->items[0].taskInfo; + } else if (pItem->type == 2) { + qTaskInfo = pRSmaInfo->items[1].taskInfo; + } else { + ASSERT(0); + } + + if (!qTaskInfo) { + smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for table:%" PRIu64, SMA_VID(pSma), pItem->suid); + return TSDB_CODE_SUCCESS; + } + + if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { + smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, + pItem->type, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile) { + memset(pIter, 0, sizeof(*pIter)); + pIter->pTFile = pTFile; + pIter->offset = TD_FILE_HEAD_SIZE; + + if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { + return TSDB_CODE_FAILED; + } + + if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { + pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; + } else { + pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; + } + + if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { + pIter->nAlloc = TD_FILE_HEAD_SIZE; + } + + pIter->pBuf = taosMemoryMalloc(pIter->nAlloc); + if (!pIter->pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->qBuf = pIter->pBuf; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish) { + STFile *pTFile = pIter->pTFile; + int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; + + if (pIter->offset >= pIter->fsize) { + *isFinish = true; + return TSDB_CODE_SUCCESS; + } + + if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { + nBytes = pIter->fsize - pIter->offset; + } + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + int32_t infoLen = 0; + taosDecodeFixedI32(pIter->qBuf, &infoLen); + if (infoLen > nBytes) { + ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE); + pIter->nAlloc = infoLen; + void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); + if (!pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->pBuf = pBuf; + pIter->qBuf = pIter->pBuf; + nBytes = infoLen; + + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + + if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + } + + pIter->offset += nBytes; + pIter->nBytes = nBytes; + pIter->nBufPos = 0; + + return TSDB_CODE_SUCCESS; +} + +static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { + while (1) { + // block iter + bool isFinish = false; + if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { + ASSERT(0); + return TSDB_CODE_FAILED; + } + if (isFinish) { + return TSDB_CODE_SUCCESS; + } + + // consume the block + int32_t qTaskInfoLenWithHead = 0; + pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); + if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FAILED; + } + + while (1) { + if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { + SRSmaQTaskInfoItem infoItem = {0}; + pIter->qBuf = taosDecodeFixedI8(pIter->qBuf, &infoItem.type); + pIter->qBuf = taosDecodeFixedI64(pIter->qBuf, &infoItem.suid); + infoItem.qTaskInfo = pIter->qBuf; + infoItem.len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); + // do the restore job + smaDebug("vgId:%d, restore the qtask info %s offset:%" PRIi64 "\n", SMA_VID(pSma), + TD_TFILE_FULL_NAME(pIter->pTFile), pIter->offset - pIter->nBytes + pIter->nBufPos); + tdRSmaQTaskInfoItemRestore(pSma, &infoItem); + + pIter->qBuf = POINTER_SHIFT(pIter->qBuf, infoItem.len); + pIter->nBufPos += qTaskInfoLenWithHead; + + if ((pIter->nBufPos + RSMA_QTASKINFO_HEAD_LEN) >= pIter->nBytes) { + // prepare and load next block in the file + pIter->offset -= (pIter->nBytes - pIter->nBufPos); + break; + } + + pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); + continue; + } + // prepare and load next block in the file + pIter->offset -= (pIter->nBytes - pIter->nBufPos); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName) { + tdGetVndFileName(vid, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName); } static void *tdRSmaPersistExec(void *param) { @@ -704,6 +954,7 @@ static void *tdRSmaPersistExec(void *param) { SRSmaStat *pRSmaStat = param; SSma *pSma = pRSmaStat->pSma; STfs *pTfs = pSma->pVnode->pTfs; + int32_t vid = SMA_VID(pSma); int64_t toffset = 0; bool isFileCreated = false; @@ -716,26 +967,16 @@ static void *tdRSmaPersistExec(void *param) { goto _end; } - STFile tFile = {0}; - int32_t vid = SMA_VID(pSma); - + STFile tFile = {0}; while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - -#if 0 - smaDebug("table %" PRIi64 " sleep 15s start ...", pRSmaInfo->items[0].pRsmaInfo->suid); - for (int32_t i = 15; i > 0; --i) { - taosSsleep(1); - smaDebug("table %" PRIi64 " countdown %d", pRSmaInfo->items[0].pRsmaInfo->suid, i); - } - smaDebug("table %" PRIi64 " sleep 15s end ...", pRSmaInfo->items[0].pRsmaInfo->suid); -#endif for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; if (!taskInfo) { smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); continue; } + char *pOutput = NULL; int32_t len = 0; int8_t type = (int8_t)(i + 1); @@ -743,69 +984,77 @@ static void *tdRSmaPersistExec(void *param) { smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1, terrstr(terrno)); goto _err; - } else { - if (!pOutput) { - smaDebug("vgId:%d, table %" PRIi64 - " level %d serialize rsma task success but no output(len %d) and no need to persist", - vid, pRSmaInfo->suid, i + 1, len); - continue; - } else if (len <= 0) { - smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and no need to persist", - vid, pRSmaInfo->suid, i + 1, len); - taosMemoryFree(pOutput); - } - smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and need persist", vid, - pRSmaInfo->suid, i + 1, len); -#if 1 - if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) { - smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid, - i + 1, terrstr(terrno)); - } else { - smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1); - } -#endif } + if (!pOutput || len <= 0) { + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success but no output(len %d), not persist", + vid, pRSmaInfo->suid, i + 1, len); + taosMemoryFreeClear(pOutput); + continue; + } + + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d, need persist", vid, + pRSmaInfo->suid, i + 1, len); +#if 0 + if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) { + smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid, + i + 1, terrstr(terrno)); + } else { + smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1); + } +#endif if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); tdInitTFile(&tFile, pTfs, qTaskInfoFName); tdCreateTFile(&tFile, pTfs, true, -1); isFileCreated = true; } - len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid)); - tdAppendTFile(&tFile, &len, sizeof(len), &toffset); - tdAppendTFile(&tFile, &type, sizeof(type), &toffset); - tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset); + + char tmpBuf[RSMA_QTASKINFO_HEAD_LEN] = {0}; + void *pTmpBuf = &tmpBuf; + int32_t headLen = 0; + headLen += taosEncodeFixedI32(&pTmpBuf, len + RSMA_QTASKINFO_HEAD_LEN); + headLen += taosEncodeFixedI8(&pTmpBuf, type); + headLen += taosEncodeFixedI64(&pTmpBuf, pRSmaInfo->suid); + + ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN); + tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset); + smaDebug("vgId:%d, table %" PRIi64 " level %d head part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid, + i + 1, headLen, toffset); tdAppendTFile(&tFile, pOutput, len, &toffset); + smaDebug("vgId:%d, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid, + i + 1, len, toffset); taosMemoryFree(pOutput); } + infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); } _normal: if (isFileCreated) { if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); + smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), + tstrerror(terrno)); tdCloseTFile(&tFile); tdRemoveTFile(&tFile); goto _err; } else { - smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_FILE_FULL_NAME(&tFile)); + smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile)); } tdCloseTFile(&tFile); char newFName[TSDB_FILENAME_LEN]; - strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); + strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); - if (taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName) != 0) { - smaError("vgId:%d, failed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) { + smaError("vgId:%d, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); goto _err; } else { - smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); } } goto _end; @@ -841,13 +1090,14 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INACTIVE, TASK_TRIGGER_STAT_ACTIVE)) { - smaDebug("persist task is active again"); + smaDebug("vgId:%d, persist task is active again", SMA_VID(pRSmaStat->pSma)); } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_CANCELLED, TASK_TRIGGER_STAT_FINISHED)) { - smaDebug(" persist task is cancelled and set finished"); + smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma)); } else { - smaWarn("persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)), + SMA_VID(pRSmaStat->pSma)); ASSERT(0); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); @@ -864,7 +1114,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *pRSmaStat = param; - int8_t tmrStat = + + int8_t tmrStat = atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); switch (tmrStat) { case TASK_TRIGGER_STAT_ACTIVE: { @@ -872,7 +1123,7 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_CANCELLED, TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("rsma persistence start since active"); + smaDebug("vgId:%d, rsma persistence start since active", SMA_VID(pRSmaStat->pSma)); // start persist task tdRSmaPersistTask(pRSmaStat); @@ -895,252 +1146,6 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { } break; default: { smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); - ASSERT(0); } break; } } - -int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { - SVnode *pVnode = pSma->pVnode; - - // step 1: iterate all stables to restore the rsma env - - SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); - if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { - smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr()); - return TSDB_CODE_FAILED; - } - - if (taosArrayGetSize(suidList) == 0) { - smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode)); - return TSDB_CODE_SUCCESS; - } - - SMetaReader mr = {0}; - metaReaderInit(&mr, SMA_META(pSma), 0); - for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - smaDebug("suid [%d] is %" PRIi64, i, suid); - if (metaGetTableEntryByUid(&mr, suid) < 0) { - smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); - goto _err; - } - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == suid); - if (TABLE_IS_ROLLUP(mr.me.flags)) { - SRSmaParam *param = &mr.me.stbEntry.rsmaParam; - for (int i = 0; i < 2; ++i) { - smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode), - suid, i, param->maxdelay[i], i, param->watermark[i]); - } - if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { - smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); - goto _err; - } - } - } - - // step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore - STFile tFile = {0}; - char qTaskInfoFName[TSDB_FILENAME_LEN]; - - tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName); - if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { - goto _err; - } - if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { - goto _err; - } - SRSmaQTaskFIter fIter = {0}; - if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { - goto _err; - } - SRSmaQTaskInfoItem infoItem = {0}; - bool isEnd = false; - int32_t code = 0; - while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) { - if (isEnd) { - break; - } - if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) { - break; - } - } - tdRSmaQTaskInfoIterDestroy(&fIter); - - if (code < 0) { - goto _err; - } - - metaReaderClear(&mr); - taosArrayDestroy(suidList); - return TSDB_CODE_SUCCESS; -_err: - ASSERT(0); - metaReaderClear(&mr); - taosArrayDestroy(suidList); - smaError("failed to restore rsma info since %s", terrstr()); - return TSDB_CODE_FAILED; -} - -static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) { - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); - SRSmaInfo *pRSmaInfo = NULL; - void *qTaskInfo = NULL; - - pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid)); - - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); - return TSDB_CODE_SUCCESS; - } - - if (infoItem->type == 1) { - qTaskInfo = pRSmaInfo->items[0].taskInfo; - } else if (infoItem->type == 2) { - qTaskInfo = pRSmaInfo->items[1].taskInfo; - } else { - ASSERT(0); - } - - if (!qTaskInfo) { - smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid); - return TSDB_CODE_SUCCESS; - } - - if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) { - smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid, - infoItem->type, terrstr(terrno)); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid, - infoItem->type); - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) { - memset(pIter, 0, sizeof(*pIter)); - pIter->pTFile = pTFile; - pIter->offset = TD_FILE_HEAD_SIZE; - - if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { - return TSDB_CODE_FAILED; - } - - if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { - pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; - } else { - pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; - } - - if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { - pIter->nAlloc = TD_FILE_HEAD_SIZE; - } - - pIter->buf = taosMemoryMalloc(pIter->nAlloc); - if (!pIter->buf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) { - STFile *pTFile = pIter->pTFile; - int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; - - if (pIter->offset >= pIter->fsize) { - *isFinish = true; - return TSDB_CODE_SUCCESS; - } - - if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { - nBytes = pIter->fsize - pIter->offset; - } - - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { - ASSERT(0); - return TSDB_CODE_FAILED; - } - - if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { - ASSERT(0); - return TSDB_CODE_FAILED; - } - - int32_t infoLen = 0; - taosDecodeFixedI32(pIter->buf, &infoLen); - if (infoLen > nBytes) { - ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE); - pIter->nAlloc = infoLen; - void *pBuf = taosMemoryRealloc(pIter->buf, infoLen); - if (!pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - pIter->buf = pBuf; - nBytes = infoLen; - - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { - ASSERT(0); - return TSDB_CODE_FAILED; - } - - if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) { - ASSERT(0); - return TSDB_CODE_FAILED; - } - } - - pIter->offset += nBytes; - pIter->nBytes = nBytes; - pIter->nBufPos = 0; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) { - while (1) { - // block iter - bool isFinish = false; - if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { - ASSERT(0); - return TSDB_CODE_FAILED; - } - if (isFinish) { - *isEnd = true; - return TSDB_CODE_SUCCESS; - } - - // consume the block - int32_t qTaskInfoLenWithHead = 0; - pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); - if (qTaskInfoLenWithHead < 0) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return TSDB_CODE_FAILED; - } - while (1) { - if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { - pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type); - pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid); - pItem->qTaskInfo = pIter->buf; - pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); - // do the restore job - printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset); - - pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len); - pIter->nBufPos += qTaskInfoLenWithHead; - - pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead); - continue; - } - // prepare and load next block in the file - pIter->offset -= (pIter->nBytes - pIter->nBufPos); - break; - } - } - - return TSDB_CODE_SUCCESS; -} diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 1f60da0b0a..5f78aadddf 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -22,7 +22,6 @@ #define TD_FILE_INIT_MAGIC 0xFFFFFFFF - static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo); static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo); @@ -46,7 +45,7 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { } int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) { - ASSERT(TD_FILE_OPENED(pTFile)); + ASSERT(TD_TFILE_OPENED(pTFile)); int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte); if (nwrite < nbyte) { @@ -58,9 +57,9 @@ int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) { } int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { - ASSERT(TD_FILE_OPENED(pTFile)); + ASSERT(TD_TFILE_OPENED(pTFile)); - int64_t loffset = taosLSeekFile(TD_FILE_PFILE(pTFile), offset, whence); + int64_t loffset = taosLSeekFile(TD_TFILE_PFILE(pTFile), offset, whence); if (loffset < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -70,12 +69,12 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { } int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) { - ASSERT(TD_FILE_OPENED(pTFile)); + ASSERT(TD_TFILE_OPENED(pTFile)); return taosFStatFile(pTFile->pFile, size, NULL); } int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { - ASSERT(TD_FILE_OPENED(pTFile)); + ASSERT(TD_TFILE_OPENED(pTFile)); int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte); if (nread < 0) { @@ -108,7 +107,7 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) { char buf[TD_FILE_HEAD_SIZE] = "\0"; uint32_t _version; - ASSERT(TD_FILE_OPENED(pTFile)); + ASSERT(TD_TFILE_OPENED(pTFile)); if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { return -1; @@ -133,7 +132,7 @@ void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) { } int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) { - ASSERT(TD_FILE_OPENED(pTFile)); + ASSERT(TD_TFILE_OPENED(pTFile)); int64_t toffset; @@ -141,6 +140,11 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) return -1; } +#if 1 + smaDebug("append to file %s, offset:%" PRIi64 " + nbyte:%" PRIi64 " =%" PRIi64, TD_TFILE_FULL_NAME(pTFile), toffset, + nbyte, toffset + nbyte); +#endif + ASSERT(pTFile->info.fsize == toffset); if (offset) { @@ -157,9 +161,9 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) } int32_t tdOpenTFile(STFile *pTFile, int flags) { - ASSERT(!TD_FILE_OPENED(pTFile)); + ASSERT(!TD_TFILE_OPENED(pTFile)); - pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), flags); + pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), flags); if (pTFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -169,9 +173,9 @@ int32_t tdOpenTFile(STFile *pTFile, int flags) { } void tdCloseTFile(STFile *pTFile) { - if (TD_FILE_OPENED(pTFile)) { + if (TD_TFILE_OPENED(pTFile)) { taosCloseFile(&pTFile->pFile); - TD_FILE_SET_CLOSED(pTFile); + TD_TFILE_SET_CLOSED(pTFile); } } @@ -183,8 +187,8 @@ int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { char fullname[TSDB_FILENAME_LEN]; SDiskID did = {0}; - TD_FILE_SET_STATE(pTFile, TD_FILE_STATE_OK); - TD_FILE_SET_CLOSED(pTFile); + TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK); + TD_TFILE_SET_CLOSED(pTFile); memset(&(pTFile->info), 0, sizeof(pTFile->info)); pTFile->info.magic = TD_FILE_INIT_MAGIC; @@ -202,18 +206,18 @@ int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) { ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); - pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pTFile->pFile == NULL) { if (errno == ENOENT) { // Try to create directory recursively - char *s = strdup(TD_FILE_REL_NAME(pTFile)); - if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_FILE_DID(pTFile)) < 0) { + char *s = strdup(TD_TFILE_REL_NAME(pTFile)); + if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) { taosMemoryFreeClear(s); return -1; } taosMemoryFreeClear(s); - pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pTFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -240,7 +244,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp return 0; } -int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } +int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); } // smaXXXUtil ================ // ... \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 4d73dbc406..57d7386667 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -152,14 +152,14 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { return pVnode; _err: - if (pVnode->pSma) smaClose(pVnode->pSma); + if (pVnode->pSma) smaCloseEnv(pVnode->pSma); if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); + if (pVnode->pSma) smaCloseEx(pVnode->pSma); if (pVnode->pMeta) metaClose(pVnode->pMeta); - tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); return NULL; From cbd581c431238ae432287ec1776109748b01f4eb Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 28 Jun 2022 14:11:44 +0800 Subject: [PATCH 15/18] feat: add taosudf.h to package files and install files --- cmake/cmake.install | 1 + packaging/check_package.sh | 2 +- packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ packaging/tools/install.sh | 3 ++- packaging/tools/install_arbi.sh | 3 ++- packaging/tools/install_client.sh | 3 ++- packaging/tools/make_install.sh | 9 +++++---- packaging/tools/makearbi.sh | 2 +- packaging/tools/makeclient.sh | 2 +- packaging/tools/makepkg.sh | 2 +- packaging/tools/post.sh | 3 ++- packaging/tools/preun.sh | 1 + packaging/tools/remove.sh | 1 + packaging/tools/remove_arbi.sh | 2 ++ packaging/tools/remove_client.sh | 1 + 17 files changed, 27 insertions(+), 12 deletions(-) diff --git a/cmake/cmake.install b/cmake/cmake.install index b2421fac25..27edd8e27a 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${TD_SOURCE_DIR}/include/client/taos.h DESTINATION include) INSTALL(FILES ${TD_SOURCE_DIR}/include/util/taoserror.h DESTINATION include) + INSTALL(FILES ${TD_SOURCE_DIR}/include/libs/function/taosudf.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) diff --git a/packaging/check_package.sh b/packaging/check_package.sh index 81abff57a5..e728c6455a 100644 --- a/packaging/check_package.sh +++ b/packaging/check_package.sh @@ -170,7 +170,7 @@ function check_lib_path() { function check_header_path() { # check all header - header_dir=("taos.h" "taosdef.h" "taoserror.h") + header_dir=("taos.h" "taosdef.h" "taoserror.h" "taosudf.h") for i in "${header_dir[@]}";do check_link ${inc_link_dir}/$i done diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index c01db74701..501398f350 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -29,6 +29,7 @@ else ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 5a14aea4ec..043c1456b8 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -70,6 +70,7 @@ cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_pat cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples #cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector #cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index d61d12932f..f440f72aa2 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -77,6 +77,7 @@ cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driv cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include #cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector @@ -201,6 +202,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${log_link_dir} || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index d730cc0d8a..7c7d5477cf 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -314,11 +314,12 @@ function install_jemalloc() { } function install_header() { - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h } function add_newHostname_to_hosts() { diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index e3c63965d4..2863640153 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -115,11 +115,12 @@ function install_bin() { } function install_header() { - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h } function install_jemalloc() { diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 5f449e5d91..0c86877c99 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -148,11 +148,12 @@ function install_lib() { } function install_header() { - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h } function install_jemalloc() { diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 6726b3fe2d..59be60f8fc 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -349,16 +349,17 @@ function install_lib() { function install_header() { if [ "$osType" != "Darwin" ]; then - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h || : - ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h \ + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : + ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h else - ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h \ + ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ ${install_main_dir}/include || - ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h \ + ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ ${install_main_2_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* || ${csudo}chmod 644 ${install_main_2_dir}/include/* diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh index 65a6dae9a4..cfae2b4a46 100755 --- a/packaging/tools/makearbi.sh +++ b/packaging/tools/makearbi.sh @@ -36,7 +36,7 @@ fi bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" install_files="${script_dir}/install_arbi.sh" -#header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h" +#header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 9e7f013006..0f1080521e 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -62,7 +62,7 @@ else lib_files="${build_dir}/lib/libtaos.${version}.dylib" fi -header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h" +header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index d0fbf0c4d7..0bc11b99b3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -93,7 +93,7 @@ else fi lib_files="${build_dir}/lib/libtaos.so.${version}" -header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h" +header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 93849dd4eb..2d744233ba 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -81,10 +81,11 @@ function kill_taosd() { } function install_include() { - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h|| : + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : ${csudo}ln -s ${inc_dir}/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${inc_dir}/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${inc_dir}/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h } function install_lib() { diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 2f35e41a48..8dee9da988 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -128,6 +128,7 @@ ${csudo}rm -f ${cfg_link_dir}/*.new || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : +${csudo}rm -f ${inc_link_dir}/taosudf.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 3b043f44a3..413de17ee6 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -84,6 +84,7 @@ function clean_header() { ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taosdef.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : + ${csudo} rm -f ${inc_link_dir}/taosudf.h || : } function clean_config() { diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh index 0a1162cd7a..c95c579d30 100755 --- a/packaging/tools/remove_arbi.sh +++ b/packaging/tools/remove_arbi.sh @@ -59,6 +59,8 @@ function clean_header() { ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taosudf.h || : + } function clean_log() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index f2cbccb45f..54f7a949f3 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -54,6 +54,7 @@ function clean_header() { ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + ${csudo}rm -f ${inc_link_dir}/taosudf.h || : } function clean_config() { From 29e69878552bee430d113bdc6a02ccccef62fd10 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 28 Jun 2022 14:12:49 +0800 Subject: [PATCH 16/18] other: merge 3.0 --- source/dnode/vnode/src/inc/sma.h | 3 +-- source/dnode/vnode/src/sma/smaRollup.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 63ac6397bd..23ad70bad3 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -28,8 +28,7 @@ extern "C" { #define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) #define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -// #define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define smaDebug(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8423654eee..f68fc4f30c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,7 +15,7 @@ #include "sma.h" -#define RSMA_QTASKINFO_PERSIST_MS 5000 // 7200000 +#define RSMA_QTASKINFO_PERSIST_MS 7200000 #define RSMA_QTASKINFO_BUFSIZE 32768 #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; From 33d1440e7bb4e130d3f5421bc9d051a1fa513245 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 28 Jun 2022 14:23:45 +0800 Subject: [PATCH 17/18] refactor(sync): adjust trace log --- source/dnode/mnode/impl/src/mndMain.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/sync/src/syncMain.c | 33 ++++++++++++++++++-------- source/libs/sync/src/syncSnapshot.c | 7 +++--- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index b66082dd98..cb2f2aef07 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndPrivilege.h" #include "mndBnode.h" #include "mndCluster.h" #include "mndConsumer.h" @@ -27,6 +26,7 @@ #include "mndMnode.h" #include "mndOffset.h" #include "mndPerfSchema.h" +#include "mndPrivilege.h" #include "mndProfile.h" #include "mndQnode.h" #include "mndQuery.h" @@ -416,7 +416,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { char *syncNodeStr = sync2SimpleStr(pMgmt->sync); static int64_t mndTick = 0; if (++mndTick % 10 == 1) { - mTrace("vgId:%d, sync heartbeat msg:%s, %s", syncGetVgId(pMgmt->sync), TMSG_INFO(pMsg->msgType), syncNodeStr); + mTrace("vgId:%d, sync trace msg:%s, %s", syncGetVgId(pMgmt->sync), TMSG_INFO(pMsg->msgType), syncNodeStr); } if (gRaftDetailLog) { char logBuf[512] = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b998f40195..d324a76438 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -243,7 +243,7 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { char *syncNodeStr = sync2SimpleStr(pVnode->sync); static int64_t vndTick = 0; if (++vndTick % 10 == 1) { - vGTrace("vgId:%d, sync heartbeat msg:%s, %s", syncGetVgId(pVnode->sync), TMSG_INFO(pMsg->msgType), syncNodeStr); + vGTrace("vgId:%d, sync trace msg:%s, %s", syncGetVgId(pVnode->sync), TMSG_INFO(pMsg->msgType), syncNodeStr); } if (gRaftDetailLog) { char logBuf[512] = {0}; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 60f6ca7e7b..89f0fcee02 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -698,7 +698,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { } else { ret = -1; terrno = TSDB_CODE_SYN_NOT_LEADER; - sError("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); + sError("sync propose not leader, %s", syncUtilState2String(pSyncNode->state)); goto _END; } @@ -1414,46 +1414,59 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { int32_t userStrLen = strlen(str); SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } - SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + SyncIndex logLastIndex = SYNC_INDEX_INVALID; + SyncIndex logBeginIndex = SYNC_INDEX_INVALID; + if (pSyncNode->pLogStore != NULL) { + logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + } + + char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); + char* printStr = ""; + if (pCfgStr != NULL) { + printStr = pCfgStr; + } if (userStrLen < 256) { - char logBuf[128 + 256]; + char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d", + "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing); + pSyncNode->changing, pSyncNode->restoreFinish, printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } sError("%s", logBuf); } else { - int len = 128 + userStrLen; + int len = 256 + userStrLen; char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d", + "lconfig:%ld, changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, - pSyncNode->changing); + pSyncNode->changing, pSyncNode->restoreFinish, printStr); } else { snprintf(s, len, "%s", str); } sError("%s", s); taosMemoryFree(s); } + + taosMemoryFree(pCfgStr); } char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 7b15be2d14..13343111f3 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -363,8 +363,9 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { uint16_t port; syncUtilU642Addr(destId.addr, host, sizeof(host), &port); - snprintf(s, len, "%s %p laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu replica-index:%d %s:%d", - event, pSender, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, + snprintf(s, len, + "%s {%p laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu replica-index:%d %s:%d}", event, + pSender, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->privateTerm, pSender->replicaIndex, host, port); @@ -629,7 +630,7 @@ char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) uint16_t port; syncUtilU642Addr(fromId.addr, host, sizeof(host), &port); - snprintf(s, len, "%s %p start:%d ack:%d term:%lu pterm:%lu from:%s:%d laindex:%ld laterm:%lu lcindex:%ld", event, + snprintf(s, len, "%s {%p start:%d ack:%d term:%lu pterm:%lu from:%s:%d laindex:%ld laterm:%lu lcindex:%ld}", event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex); From df417c7f505b14eb73303d40f402ec3b4fdf9040 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 28 Jun 2022 14:48:40 +0800 Subject: [PATCH 18/18] fix: not restore rsma if qtaskinfo file not exist --- source/dnode/vnode/src/sma/smaRollup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index f68fc4f30c..34c3ef8fca 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -749,9 +749,17 @@ int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { goto _err; } + + if(!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) { + metaReaderClear(&mr); + taosArrayDestroy(suidList); + return TSDB_CODE_SUCCESS; + } + if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { goto _err; } + SRSmaQTaskInfoIter fIter = {0}; if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { goto _err;