From 6d952626882dbdc66a8c4cb1572b1ca69bf9330d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 19:11:28 +0800 Subject: [PATCH 01/25] enh(query): add first/last function distributed implementation --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 57 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 4 ++ 4 files changed, 64 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e86d643c0d..f01e734fde 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -138,6 +138,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_TOP_MERGE, FUNCTION_TYPE_BOTTOM_PARTIAL, FUNCTION_TYPE_BOTTOM_MERGE, + FUNCTION_TYPE_FIRST_PARTIAL, + FUNCTION_TYPE_FIRST_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index a528041964..f3bcad8ede 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -97,6 +97,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getFirstLastInfoSize(int32_t resBytes); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23940a415e..a2849d601e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -954,6 +954,41 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } +static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + // first(col_list) will be rewritten as first(col) + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + uint8_t paraType = ((SExprNode*)pPara)->resType.type; + int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; + if (isPartial) { + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The parameters of first/last can only be columns"); + } + + pFunc->node.resType = (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, + .type = TSDB_DATA_TYPE_BINARY}; + } else { + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = ((SExprNode*)pPara)->resType; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateFirstLastPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateFirstLastImpl(pFunc, pErrBuf, len, true); +} + +static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateFirstLastImpl(pFunc, pErrBuf, len, false); +} + static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -1706,6 +1741,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = firstCombine, }, + { + .name = "_first_partial", + .type = FUNCTION_TYPE_FIRST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastPartial, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = firstLastFinalize, + .combineFunc = firstCombine, + }, + { + .name = "_first_merge", + .type = FUNCTION_TYPE_FIRST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = firstLastFinalize, + .combineFunc = firstCombine, + }, { .name = "last", .type = FUNCTION_TYPE_LAST, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3d64f82eba..44418685b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2244,6 +2244,10 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) return TSDB_CODE_SUCCESS; } +int32_t getFirstLastInfoSize(int32_t resBytes) { + return resBytes + sizeof(int64_t); +} + bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); From 45ef3db78abd908abbc57a7bfbc56871381413f2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 20:53:42 +0800 Subject: [PATCH 02/25] add first function partial/merge --- source/libs/function/inc/builtinsimpl.h | 2 + source/libs/function/src/builtins.c | 4 +- source/libs/function/src/builtinsimpl.c | 82 +++++++++++++++++++++---- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f3bcad8ede..af9b8d4a10 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -93,8 +93,10 @@ int32_t diffFunction(SqlFunctionCtx *pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); +int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getFirstLastInfoSize(int32_t resBytes); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a2849d601e..180ffd1848 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1749,7 +1749,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, - .finalizeFunc = firstLastFinalize, + .finalizeFunc = firstLastPartialFinalize, .combineFunc = firstCombine, }, { @@ -1759,7 +1759,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = firstFunction, + .processFunc = firstFunctionMerge, .finalizeFunc = firstLastFinalize, .combineFunc = firstCombine, }, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 44418685b0..1bb27780bc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -72,6 +72,12 @@ typedef struct STopBotRes { STopBotResItem* pItems; } STopBotRes; +typedef struct SFirstLastRes { + bool hasResult; + int32_t bytes; + char buf[]; +} SFirstLastRes; + typedef struct SStddevRes { double result; int64_t count; @@ -2245,12 +2251,12 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } int32_t getFirstLastInfoSize(int32_t resBytes) { - return resBytes + sizeof(int64_t); + return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); - pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); + pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t); return true; } @@ -2274,12 +2280,13 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2297,7 +2304,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(buf + bytes); + TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); if (ts < startKey) { return TSDB_CODE_SUCCESS; } @@ -2313,9 +2320,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; @@ -2326,7 +2334,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // in case of descending order time stamp serial, which usually happens as the results of the nest query, // all data needs to be check. if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(buf + bytes); + TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); if (ts < endKey) { return TSDB_CODE_SUCCESS; } @@ -2342,9 +2350,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; break; @@ -2422,6 +2431,39 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput) { + pOutput->bytes = pInput->bytes; + TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes); + TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); + if (pOutput->hasResult) { + if (*tsIn > *tsOut) { + return; + } + } + *tsOut = *tsIn; + memcpy(pOutput->buf, pInput->buf, pOutput->bytes); + pOutput->hasResult = true; + return; +} + +int32_t firstFunctionMerge(SqlFunctionCtx *pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SFirstLastRes* pInputInfo = (SFirstLastRes *)varDataVal(data); + + firstLastTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -2435,6 +2477,24 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getFirstLastInfoSize(pRes->bytes); + char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pRes, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return 1; +} + int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); From 4240a38c1783804d4ee7aa0404fca0e305e8ccdc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 20:56:33 +0800 Subject: [PATCH 03/25] fix bugs --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1bb27780bc..280f0217b9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2471,8 +2471,8 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; - char* in = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes); return pResInfo->numOfRes; } From 763e8bd6aef99264dc6608df5881738be11d9790 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 21:17:55 +0800 Subject: [PATCH 04/25] enable first function split --- source/libs/function/src/builtins.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 180ffd1848..3bfb60ec7c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -956,7 +956,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { // first(col_list) will be rewritten as first(col) - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + if (2 != LIST_LENGTH(pFunc->pParameterList)) { //input has two params c0,ts, is this a bug? return TSDB_CODE_SUCCESS; } @@ -1739,6 +1739,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = firstFunction, .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_first_partial", + .pMergeFunc = "_first_merge", .combineFunc = firstCombine, }, { From d4a38245a20d6296716200b253a7b8ec5a51abf3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 21:21:46 +0800 Subject: [PATCH 05/25] add some check --- source/libs/function/src/builtinsimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 280f0217b9..032333228a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2432,6 +2432,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput) { + if (!pInput->hasResult) { + return; + } pOutput->bytes = pInput->bytes; TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes); TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); From 4481bb4e5cc4fdbe8aa7923124e5c552b5baafc5 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Jun 2022 21:29:00 +0800 Subject: [PATCH 06/25] fix: error in schemaless with same timestamp --- source/client/src/clientSml.c | 26 +++++++++++++------------- source/client/test/smlTest.cpp | 26 +++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index aaa13ed266..0acfaf00f2 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1297,8 +1297,8 @@ static int32_t smlKvTimeArrayCompare(const void* key1, const void* key2) { static int32_t smlKvTimeHashCompare(const void* key1, const void* key2) { SHashObj *s1 = *(SHashObj **)key1; SHashObj *s2 = *(SHashObj **)key2; - SSmlKv *kv1 = (SSmlKv *)taosHashGet(s1, TS, TS_LEN); - SSmlKv *kv2 = (SSmlKv *)taosHashGet(s2, TS, TS_LEN); + SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); + SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); if (kv1->i < kv2->i) { @@ -1316,17 +1316,17 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(p == NULL){ taosArrayPush(oneTable->cols, &cols); }else{ // to make the sort stable for update data - SArray *sa = (SArray *)p; - SSmlKv *cur = (SSmlKv*)taosArrayGet(sa, 0); - SSmlKv *dCur = (SSmlKv*)taosArrayGet(cols, 0); + SArray *sa = *(SArray **)p; + SSmlKv *cur = (SSmlKv*)taosArrayGetP(sa, 0); + SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); if(cur->i > dCur->i){ taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); }else{ ASSERT(cur->i == dCur->i); int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; for(; index < taosArrayGetSize(oneTable->cols); index++){ - SArray *tmp = (SArray *)taosArrayGet(oneTable->cols, index); - SSmlKv *curTs = (SSmlKv*)taosArrayGet(tmp, 0); + SArray *tmp = (SArray *)taosArrayGetP(oneTable->cols, index); + SSmlKv *curTs = (SSmlKv*)taosArrayGetP(tmp, 0); if(curTs->i > dCur->i){ break; } @@ -1351,22 +1351,22 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col if(p == NULL){ taosArrayPush(oneTable->cols, &kvHash); }else{ // to make the sort stable for update data - SHashObj *sa = (SHashObj *)p; - SSmlKv *cur = (SSmlKv *)taosHashGet(sa, TS, TS_LEN); - SSmlKv *dCur = (SSmlKv*)taosArrayGet(cols, 0); + SHashObj *sa = *(SHashObj **)p; + SSmlKv *cur = *(SSmlKv **)taosHashGet(sa, TS, TS_LEN); + SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); if(cur->i > dCur->i){ taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); }else{ ASSERT(cur->i == dCur->i); int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; for(; index < taosArrayGetSize(oneTable->cols); index++){ - SHashObj *tmp = (SHashObj *)taosArrayGet(oneTable->cols, index); - SSmlKv *curTs = (SSmlKv *)taosHashGet(tmp, TS, TS_LEN); + SHashObj *tmp = (SHashObj *)taosArrayGetP(oneTable->cols, index); + SSmlKv *curTs = *(SSmlKv **)taosHashGet(tmp, TS, TS_LEN); if(curTs->i > dCur->i){ break; } } - taosArrayInsert(oneTable->cols, index, &cols); + taosArrayInsert(oneTable->cols, index, &kvHash); } } return TSDB_CODE_SUCCESS; diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 4ad73a6424..fffb03d9a5 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1260,4 +1260,28 @@ TEST(testCase, sml_16368_Test) { pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); -}*/ +} + +TEST(testCase, sml_dup_time_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists dup_time schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + //"test_ms,t0=t c0=f 1626006833641", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" + }; + pRes = taos_query(taos, "use dup_time"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + ASSERT_EQ(taos_errno(pRes), 0); + taos_free_result(pRes); +} +*/ From 877cb761f1b4fdc316078b47ffa0abbee1e17783 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 09:43:22 +0800 Subject: [PATCH 07/25] refactor: mnode worker --- include/dnode/mnode/mnode.h | 1 + source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 14 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 218 ++++++++++---------- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 88 ++++---- 5 files changed, 161 insertions(+), 165 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index ab090940f2..83183422c2 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -83,6 +83,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); +int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); /** * @brief Generate machine code diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 22691300bc..2443493d0b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -53,20 +53,18 @@ void mmRelease(SMnodeMgmt *pMgmt); SArray *mmGetMsgHandles(); int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt); -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); +int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index c47988e1ef..f151a09b94 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -20,11 +20,6 @@ void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); } -void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - pInfo->isMnode = 1; - mndGetLoad(pMgmt->pMnode, &pInfo->load); -} - int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonMmInfo mmInfo = {0}; mmGetMonitorInfo(pMgmt, &mmInfo); @@ -50,6 +45,11 @@ int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } +void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { + pInfo->isMnode = 1; + mndGetLoad(pMgmt->pMnode, &pInfo->load); +} + int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonMloadInfo mloads = {0}; mmGetMnodeLoads(pMgmt, &mloads); @@ -129,114 +129,114 @@ SArray *mmGetMsgHandles() { SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 0f3c06cb3a..cb942e8d1c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -43,7 +43,6 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica = 1; pOption->selfIndex = 0; @@ -54,8 +53,8 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - pOption->deploy = false; pOption->standby = false; + pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; @@ -105,7 +104,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutRpcMsgToQueue; + pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutMsgToQueue; pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 53943b61b0..6da9bfc528 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -26,7 +26,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { tmsgSendRsp(&rsp); } -static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; dTrace("msg:%p, get from mnode queue", pMsg); @@ -53,11 +53,10 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; - dTrace("msg:%p, get from mnode-sync queue", pMsg); - pMsg->info.node = pMgmt->pMnode; + dTrace("msg:%p, get from mnode-sync queue", pMsg); SMsgHead *pHead = pMsg->pCont; pHead->contLen = ntohl(pHead->contLen); @@ -70,64 +69,63 @@ static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); - taosWriteQitem(pWorker->queue, pMsg); - return 0; +static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pMsg) { + if (mmAcquire(pMgmt) == 0) { + dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + taosWriteQitem(pWorker->queue, pMsg); + mmRelease(pMgmt); + return 0; + } else { + dTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(), + TMSG_INFO(pMsg->msgType)); + return -1; + } } -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); +inline int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); } -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); +inline int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); +inline int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - mndPreprocessQueryMsg(pMgmt->pMnode, pMsg); - - return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); +inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { + dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); + return -1; + } + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } -int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); +inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } -int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { +inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dTrace("msg:%p, is created", pMsg); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); switch (qtype) { case WRITE_QUEUE: - dTrace("msg:%p, is created and will put into vnode-write queue", pMsg); - taosWriteQitem(pMgmt->writeWorker.queue, pMsg); - return 0; + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); case QUERY_QUEUE: - dTrace("msg:%p, is created and will put into vnode-query queue", pMsg); - taosWriteQitem(pMgmt->queryWorker.queue, pMsg); - return 0; - + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); case READ_QUEUE: - dTrace("msg:%p, is created and will put into vnode-read queue", pMsg); - taosWriteQitem(pMgmt->readWorker.queue, pMsg); - return 0; + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); case SYNC_QUEUE: - if (mmAcquire(pMgmt) == 0) { - dTrace("msg:%p, is created and will put into vnode-sync queue", pMsg); - taosWriteQitem(pMgmt->syncWorker.queue, pMsg); - mmRelease(pMgmt); - return 0; - } else { - return -1; - } + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); default: terrno = TSDB_CODE_INVALID_PARA; + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pMsg->pCont); return -1; } } @@ -137,7 +135,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads, .name = "mnode-query", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { @@ -149,7 +147,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeReadThreads, .max = tsNumOfMnodeReadThreads, .name = "mnode-read", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) { @@ -161,7 +159,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-write", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) { @@ -173,7 +171,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-sync", - .fp = (FItem)mmProcessSyncQueue, + .fp = (FItem)mmProcessSyncMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { @@ -185,7 +183,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-monitor", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { From 01521a2541af1aa7cd62eb790163ce89407f5807 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 10:41:15 +0800 Subject: [PATCH 08/25] refactor: mnode file --- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 6 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 84 ++++++++------------- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 24 +++--- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 8 +- 5 files changed, 47 insertions(+), 77 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 2443493d0b..5dd1a5b3b8 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -34,16 +34,14 @@ typedef struct SMnodeMgmt { SSingleWorker writeWorker; SSingleWorker syncWorker; SSingleWorker monitorWorker; - SReplica replicas[TSDB_MAX_REPLICA]; - int8_t replica; bool stopped; int32_t refCount; TdThreadRwlock lock; } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed); +int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); // mmInt.c int32_t mmAcquire(SMnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 478d6abd52..e24ebcb9e2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { +int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -52,45 +52,36 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { } *pDeployed = deployed->valueint; - cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); - if (mnodes != NULL) { - if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", file); + cJSON *id = cJSON_GetObjectItem(root, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); goto _OVER; } - - pMgmt->replica = cJSON_GetArraySize(mnodes); - if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); - goto _OVER; - } - - for (int32_t i = 0; i < pMgmt->replica; ++i) { - cJSON *node = cJSON_GetArrayItem(mnodes, i); - if (node == NULL) break; - - SReplica *pReplica = &pMgmt->replicas[i]; - - cJSON *id = cJSON_GetObjectItem(node, "id"); - if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto _OVER; - } + if (pReplica) { pReplica->id = id->valueint; + } + } - cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto _OVER; - } + cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } - cJSON *port = cJSON_GetObjectItem(node, "port"); - if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto _OVER; - } - pReplica->port = port->valueint; + cJSON *port = cJSON_GetObjectItem(root, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; } } @@ -106,7 +97,7 @@ _OVER: return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed) { +int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); @@ -124,26 +115,11 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed) { char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - - int8_t replica = (pMsg != NULL ? pMsg->replica : pMgmt->replica); - if (replica > 0) { - len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - for (int32_t i = 0; i < replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - if (pMsg != NULL) { - pReplica = &pMsg->replicas[i]; - } - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }],\n"); - } - } + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); len += snprintf(content + len, maxLen - len, "}\n"); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f151a09b94..124876d718 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -90,7 +90,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SMnodeMgmt mgmt = {0}; mgmt.path = pInput->path; mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq, deployed) != 0) { + if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index cb942e8d1c..7e73e53481 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -27,7 +27,7 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { SMnodeMgmt mgmt = {0}; mgmt.path = pInput->path; - if (mmReadFile(&mgmt, required) != 0) { + if (mmReadFile(&mgmt, NULL, required) != 0) { return -1; } @@ -52,23 +52,17 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu tstrncpy(pReplica->fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { pOption->standby = false; pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pMgmt->replica > 0) { + if (pReplica->id > 0) { pOption->standby = true; pOption->replica = 1; pOption->selfIndex = 0; - SReplica *pReplica = &pOption->replicas[0]; - for (int32_t i = 0; i < pMgmt->replica; ++i) { - if (pMgmt->replicas[i].id != pMgmt->pData->dnodeId) continue; - pReplica->id = pMgmt->replicas[i].id; - pReplica->port = pMgmt->replicas[i].port; - memcpy(pReplica->fqdn, pMgmt->replicas[i].fqdn, TSDB_FQDN_LEN); - } + memcpy(&pOption->replicas[0], pReplica, sizeof(SReplica)); } } @@ -108,8 +102,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - if (mmReadFile(pMgmt, &deployed) != 0) { + bool deployed = false; + SReplica replica = {0}; + if (mmReadFile(pMgmt, &replica, &deployed) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; @@ -122,7 +117,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &option); + mmBuildOptionForOpen(pMgmt, &replica, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -140,8 +135,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || pMgmt->replica > 0) { - pMgmt->replica = 0; + if (!deployed || replica.id > 0) { deployed = true; if (mmWriteFile(pMgmt, NULL, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 528beb280b..498af36786 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -123,10 +123,12 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); } else { dInfo("node:%s, has been created", pWrapper->name); - (void)dmOpenNode(pWrapper); - (void)dmStartNode(pWrapper); - pWrapper->required = true; + code = dmOpenNode(pWrapper); + if (code != 0) { + code = dmStartNode(pWrapper); + } pWrapper->deployed = true; + pWrapper->required = true; pWrapper->proc.ptype = pDnode->ptype; } From b5bc9aa2bae4dc9df60577b0b6d0cacec53f0107 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Jun 2022 10:44:34 +0800 Subject: [PATCH 09/25] fix rpc retry error --- source/libs/transport/src/transCli.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d1debc6af6..4852dcfb54 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -983,6 +983,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SEpSet epSet = {0}; tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); pCtx->epSet = epSet; + if (!transEpSetIsEqual(&epSet, &pCtx->epSet)) { + pCtx->retryCount = 0; + } } addConnToPool(pThrd->pool, pConn); tTrace("use remote epset, current in use: %d, retry count:%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, From 589b30422f180cc990a371cc4d0fdd36843d26d4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 10:51:00 +0800 Subject: [PATCH 10/25] add last function splitting --- include/libs/function/functionMgt.h | 2 ++ source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e787639897..b5641f2d07 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -140,6 +140,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_BOTTOM_MERGE, FUNCTION_TYPE_FIRST_PARTIAL, FUNCTION_TYPE_FIRST_MERGE, + FUNCTION_TYPE_LAST_PARTIAL, + FUNCTION_TYPE_LAST_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index af9b8d4a10..7db1396603 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -95,6 +95,7 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +int32_t lastFunctionMerge(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0a6d99c52d..7a06304487 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1776,6 +1776,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = lastCombine, }, + { + .name = "_last_partial", + .type = FUNCTION_TYPE_LAST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastPartial, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunction, + .finalizeFunc = firstLastPartialFinalize, + .combineFunc = lastCombine, + }, + { + .name = "_last_merge", + .type = FUNCTION_TYPE_LAST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = firstLastFinalize, + .combineFunc = lastCombine, + }, { .name = "twa", .type = FUNCTION_TYPE_TWA, From a3f12736407af1bbe8ed75435fde928b70167ef9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 10:58:36 +0800 Subject: [PATCH 11/25] refactor: mnode sync --- include/dnode/mnode/mnode.h | 4 +--- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 4 +++- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 15 ++++----------- source/dnode/mnode/impl/inc/mndInt.h | 14 ++++++-------- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- source/dnode/mnode/impl/src/mndMain.c | 4 +--- source/dnode/mnode/impl/src/mndSync.c | 18 +++++++++--------- 8 files changed, 27 insertions(+), 38 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 83183422c2..99b704826d 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -32,9 +32,7 @@ typedef struct { int32_t dnodeId; bool standby; bool deploy; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; + SReplica replica; SMsgCb msgCb; } SMnodeOpt; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index e24ebcb9e2..27a35ae17a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -86,12 +86,14 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { } code = 0; - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); _OVER: if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); + if (code == 0) { + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + } terrno = code; return code; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 124876d718..e91f4b8cf4 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -126,7 +126,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SArray *mmGetMsgHandles() { int32_t code = -1; - SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); + SArray *pArray = taosArrayInit(128, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 7e73e53481..bc67b8442e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -43,13 +43,9 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica = 1; - pOption->selfIndex = 0; - - SReplica *pReplica = &pOption->replicas[0]; - pReplica->id = 1; - pReplica->port = tsServerPort; - tstrncpy(pReplica->fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->replica.id = 1; + pOption->replica.port = tsServerPort; + tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { @@ -57,12 +53,9 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SM pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { pOption->standby = true; - pOption->replica = 1; - pOption->selfIndex = 0; - memcpy(&pOption->replicas[0], pReplica, sizeof(SReplica)); + pOption->replica = *pReplica; } } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 4869a19856..16220f101a 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -76,11 +76,12 @@ typedef struct { } STelemMgmt; typedef struct { - sem_t syncSem; - int64_t sync; - bool standby; - int32_t errCode; - int32_t transId; + sem_t syncSem; + int64_t sync; + bool standby; + SReplica replica; + int32_t errCode; + int32_t transId; } SSyncMgmt; typedef struct { @@ -98,9 +99,6 @@ typedef struct SMnode { bool stopped; bool restored; bool deploy; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; char *path; int64_t checkTime; SSdb *pSdb; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73eea70195..c936c0f93d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -95,8 +95,8 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { dnodeObj.id = 1; dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; - dnodeObj.port = pMnode->replicas[0].port; - memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); + dnodeObj.port = tsServerPort; + memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 7a73d3c360..892650fa18 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -289,11 +289,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { } static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { - pMnode->replica = pOption->replica; - pMnode->selfIndex = pOption->selfIndex; - memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; + pMnode->syncMgmt.replica = pOption->replica; pMnode->syncMgmt.standby = pOption->standby; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e0b4cc6a57..e58012b1b7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -188,15 +188,15 @@ int32_t mndInitSync(SMnode *pMnode) { syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotEnable = true; - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = pMnode->replica; - pCfg->myIndex = pMnode->selfIndex; - mInfo("start to open mnode sync, replica:%d myindex:%d standby:%d", pCfg->replicaNum, pCfg->myIndex, pMgmt->standby); - for (int32_t i = 0; i < pMnode->replica; ++i) { - SNodeInfo *pNode = &pCfg->nodeInfo[i]; - tstrncpy(pNode->nodeFqdn, pMnode->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMnode->replicas[i].port; - mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + mInfo("start to open mnode sync, standby:%d", pMgmt->standby); + if (pMgmt->standby || pMgmt->replica.id > 0) { + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = 1; + pCfg->myIndex = 0; + SNodeInfo *pNode = &pCfg->nodeInfo[0]; + tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replica.port; + mInfo("fqdn:%s port:%u", pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); From 6a4fc4577126119b4ffc3d2f0285862a8b656647 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:00:22 +0800 Subject: [PATCH 12/25] fix bugs --- source/libs/function/src/builtinsimpl.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f2c5df8fc2..fe52e9c7c5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2355,12 +2355,13 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2385,9 +2386,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; } @@ -2403,9 +2404,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } From 6f27ee975e758cf4af935b0d554c740412f9c2b7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:04:36 +0800 Subject: [PATCH 13/25] enable last splitting --- source/libs/function/src/builtins.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7a06304487..1c3f274ee9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1774,6 +1774,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = lastFunction, .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_last_partial", + .pMergeFunc = "_last_merge", .combineFunc = lastCombine, }, { From d93f4c4a83d906e729cbf8fc344cdfa4c5b14254 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:10:36 +0800 Subject: [PATCH 14/25] refactor: mnode worker --- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 4 -- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 19 -------- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 48 +++++++++++++++------ 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 5dd1a5b3b8..c5c3d76f1e 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -43,10 +43,6 @@ typedef struct SMnodeMgmt { int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); -// mmInt.c -int32_t mmAcquire(SMnodeMgmt *pMgmt); -void mmRelease(SMnodeMgmt *pMgmt); - // mmHandle.c SArray *mmGetMsgHandles(); int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index bc67b8442e..b7124dfaa5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -164,22 +164,3 @@ SMgmtFunc mmGetMgmtFunc() { return mgmtFunc; } - -int32_t mmAcquire(SMnodeMgmt *pMgmt) { - int32_t code = 0; - - taosThreadRwlockRdlock(&pMgmt->lock); - if (pMgmt->stopped) { - code = -1; - } else { - atomic_add_fetch_32(&pMgmt->refCount, 1); - } - taosThreadRwlockUnlock(&pMgmt->lock); - return code; -} - -void mmRelease(SMnodeMgmt *pMgmt) { - taosThreadRwlockRdlock(&pMgmt->lock); - atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosThreadRwlockUnlock(&pMgmt->lock); -} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 6da9bfc528..17581fcef7 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -16,6 +16,25 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) { + int32_t code = 0; + + taosThreadRwlockRdlock(&pMgmt->lock); + if (pMgmt->stopped) { + code = -1; + } else { + atomic_add_fetch_32(&pMgmt->refCount, 1); + } + taosThreadRwlockUnlock(&pMgmt->lock); + return code; +} + +static inline void mmRelease(SMnodeMgmt *pMgmt) { + taosThreadRwlockRdlock(&pMgmt->lock); + atomic_sub_fetch_32(&pMgmt->refCount, 1); + taosThreadRwlockUnlock(&pMgmt->lock); +} + static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, @@ -107,27 +126,30 @@ inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - if (pMsg == NULL) return -1; - dTrace("msg:%p, is created", pMsg); - - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); + pWorker = &pMgmt->writeWorker; + break; case QUERY_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); + pWorker = &pMgmt->queryWorker; + break; case READ_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); + pWorker = &pMgmt->readWorker; + break; case SYNC_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); + pWorker = &pMgmt->syncWorker; + break; default: terrno = TSDB_CODE_INVALID_PARA; - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pMsg->pCont); - return -1; } + + if (pWorker == NULL) return -1; + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + if (pMsg == NULL) return -1; + dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + + return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { From 22f86ee52978c26d91ff137c1b552f7a4d7fa389 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 14 Jun 2022 11:15:10 +0800 Subject: [PATCH 15/25] feat(stream): partition by --- source/libs/executor/src/groupoperator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 132f93a6a5..f663f5798a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -582,6 +582,15 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { return offset; } +static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { + void *ite = NULL; + while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) { + taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList); + } + taosHashClear(pInfo->pGroupSet); + clearDiskbasedBuf(pInfo->pBuf); +} + static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; @@ -591,6 +600,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pGroupIter = taosHashIterate(pInfo->pGroupSet, pInfo->pGroupIter); if (pInfo->pGroupIter == NULL) { doSetOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); return NULL; } From 63927d15fab259f84e78e710ce1011fb3aff8ad6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:15:17 +0800 Subject: [PATCH 16/25] fix: compile err in windows --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 17581fcef7..f02f90962b 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -101,19 +101,19 @@ static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker } } -inline int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); } -inline int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -inline int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); return -1; @@ -121,11 +121,11 @@ inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } -inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } -inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { +int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: From 313d2f0e4dbcd7d647d9a8bbd511e2151f9c08bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:18:30 +0800 Subject: [PATCH 17/25] test: enable mnode/basic3.sim --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 5ffb01b2e69a2641f2ca51e28d6bea339ad58ac2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jun 2022 11:24:45 +0800 Subject: [PATCH 18/25] fix: error in schemaless with same timestamp --- source/client/src/clientSml.c | 45 +--- source/libs/parser/src/parInsertData.c | 4 + source/util/src/talgo.c | 12 +- source/util/test/taosbsearchTest.cpp | 328 +++++++++++++++++++++++++ 4 files changed, 350 insertions(+), 39 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 0acfaf00f2..c10311acee 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1312,27 +1312,11 @@ static int32_t smlKvTimeHashCompare(const void* key1, const void* key2) { static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *cols){ if(dataFormat){ - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GE); + void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); if(p == NULL){ taosArrayPush(oneTable->cols, &cols); - }else{ // to make the sort stable for update data - SArray *sa = *(SArray **)p; - SSmlKv *cur = (SSmlKv*)taosArrayGetP(sa, 0); - SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); - if(cur->i > dCur->i){ - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); - }else{ - ASSERT(cur->i == dCur->i); - int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; - for(; index < taosArrayGetSize(oneTable->cols); index++){ - SArray *tmp = (SArray *)taosArrayGetP(oneTable->cols, index); - SSmlKv *curTs = (SSmlKv*)taosArrayGetP(tmp, 0); - if(curTs->i > dCur->i){ - break; - } - } - taosArrayInsert(oneTable->cols, index, &cols); - } + }else{ + taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); } return TSDB_CODE_SUCCESS; } @@ -1347,29 +1331,14 @@ static int32_t smlDealCols(SSmlTableInfo* oneTable, bool dataFormat, SArray *col taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GE); + void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); if(p == NULL){ taosArrayPush(oneTable->cols, &kvHash); - }else{ // to make the sort stable for update data - SHashObj *sa = *(SHashObj **)p; - SSmlKv *cur = *(SSmlKv **)taosHashGet(sa, TS, TS_LEN); - SSmlKv *dCur = (SSmlKv*)taosArrayGetP(cols, 0); - if(cur->i > dCur->i){ - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); - }else{ - ASSERT(cur->i == dCur->i); - int32_t index = TARRAY_ELEM_IDX(oneTable->cols, p) + 1; - for(; index < taosArrayGetSize(oneTable->cols); index++){ - SHashObj *tmp = (SHashObj *)taosArrayGetP(oneTable->cols, index); - SSmlKv *curTs = *(SSmlKv **)taosHashGet(tmp, TS, TS_LEN); - if(curTs->i > dCur->i){ - break; - } - } - taosArrayInsert(oneTable->cols, index, &kvHash); - } + }else{ + taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } static SSmlSTableMeta* smlBuildSTableMeta(){ diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 1960073f29..2b505d4bf7 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -283,6 +283,8 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) { if (!dataBuf->ordered) { char* pBlockData = pBlocks->data; + + // todo. qsort is unstable, if timestamp is same, should get the last one qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); int32_t i = 0; @@ -350,6 +352,8 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey if (!dataBuf->ordered) { pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + + // todo. qsort is unstable, if timestamp is same, should get the last one qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 866ac5757b..5353cd9bfe 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -175,7 +175,13 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size c = compar(key, p); if (c == 0) { - break; + if (flags == TD_GT){ + lidx = midx + 1; + } else if(flags == TD_LT){ + ridx = midx - 1; + }else{ + break; + } } else if (c < 0) { ridx = midx - 1; } else { @@ -189,6 +195,10 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size return (c <= 0) ? p : (midx + 1 < nmemb ? p + size : NULL); } else if (flags == TD_LE) { return (c >= 0) ? p : (midx > 0 ? p - size : NULL); + } else if (flags == TD_GT) { + return (c < 0) ? p : (midx + 1 < nmemb ? p + size : NULL); + } else if (flags == TD_LT) { + return (c > 0) ? p : (midx > 0 ? p - size : NULL); } else { ASSERT(0); } diff --git a/source/util/test/taosbsearchTest.cpp b/source/util/test/taosbsearchTest.cpp index 0b250c9ecc..b8fa3a4476 100644 --- a/source/util/test/taosbsearchTest.cpp +++ b/source/util/test/taosbsearchTest.cpp @@ -248,6 +248,170 @@ TEST(testCase, taosbsearch_greater_or_equal) { ASSERT_EQ(pRet, nullptr); } +TEST(testCase, taosbsearch_greater) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 5); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 4 element + int array4[4] = {3, 6, 8, 11}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + // 5 element + int array5[5] = {3, 6, 8, 11, 15}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 15); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(*(int *)pRet, 15); + + key = 15; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); + + key = 17; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_GT); + ASSERT_EQ(pRet, nullptr); +} + TEST(testCase, taosbsearch_less_or_equal) { // For equal test int key = 3; @@ -411,4 +575,168 @@ TEST(testCase, taosbsearch_less_or_equal) { key = 17; pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LE); ASSERT_EQ(*(int *)pRet, 15); +} + +TEST(testCase, taosbsearch_less) { + // For equal test + int key = 3; + void *pRet = NULL; + + pRet = taosbsearch((void *)&key, NULL, 0, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + // 1 element + int array1[1] = {5}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 5); + + key = 5; + pRet = taosbsearch((void *)&key, (void *)array1, 1, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + // 2 element + int array2[2] = {3, 6}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array2, 2, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + // 3 element + int array3[3] = {3, 6, 8}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array3, 3, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + // 4 element + int array4[4] = {3, 6, 8, 11}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array4, 4, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 11); + + // 5 element + int array5[5] = {3, 6, 8, 11, 15}; + + key = 1; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 3; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(pRet, nullptr); + + key = 4; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 6; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 3); + + key = 7; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 8; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 6); + + key = 9; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 11; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 8); + + key = 13; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 15; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 11); + + key = 17; + pRet = taosbsearch((void *)&key, (void *)array5, 5, sizeof(int), compareFunc, TD_LT); + ASSERT_EQ(*(int *)pRet, 15); } \ No newline at end of file From 0dde9cc9f95c9d6aebabc8a3b4b2feb2be12ffc3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:25:11 +0800 Subject: [PATCH 19/25] fix bugs --- source/libs/function/src/builtinsimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fe52e9c7c5..b5009b7d41 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2390,6 +2390,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + pInfo->hasResult = true; pResInfo->numOfRes = 1; } break; @@ -2407,6 +2408,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } From 4e7aac8171a99b41c3c020e54046bbe95dc1d0d0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:34:51 +0800 Subject: [PATCH 20/25] fix: crash in mnode --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index f02f90962b..ee2df5f089 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -147,8 +147,9 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { if (pWorker == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } From d44c6a810bb288839d18fb68d88070d807a2a2ed Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 11:54:55 +0800 Subject: [PATCH 21/25] docs: refine keywords section and add delete keywords --- docs/en/12-taos-sql/12-keywords.md | 311 +++++++++++++++++++++++++---- docs/zh/12-taos-sql/12-keywords.md | 310 ++++++++++++++++++++++++---- 2 files changed, 536 insertions(+), 85 deletions(-) diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/12-keywords.md index ed0c96b4e4..6d40deb5a6 100644 --- a/docs/en/12-taos-sql/12-keywords.md +++ b/docs/en/12-taos-sql/12-keywords.md @@ -4,50 +4,275 @@ title: Keywords There are about 200 keywords reserved by TDengine, they can't be used as the name of database, STable or table with either upper case, lower case or mixed case. -**Keywords List** +## Keywords List -| | | | | | -| ----------- | ---------- | --------- | ---------- | ------------ | -| ABORT | CREATE | IGNORE | NULL | STAR | -| ACCOUNT | CTIME | IMMEDIATE | OF | STATE | -| ACCOUNTS | DATABASE | IMPORT | OFFSET | STATEMENT | -| ADD | DATABASES | IN | OR | STATE_WINDOW | -| AFTER | DAYS | INITIALLY | ORDER | STORAGE | -| ALL | DBS | INSERT | PARTITIONS | STREAM | -| ALTER | DEFERRED | INSTEAD | PASS | STREAMS | -| AND | DELIMITERS | INT | PLUS | STRING | -| AS | DESC | INTEGER | PPS | SYNCDB | -| ASC | DESCRIBE | INTERVAL | PRECISION | TABLE | -| ATTACH | DETACH | INTO | PREV | TABLES | -| BEFORE | DISTINCT | IS | PRIVILEGE | TAG | -| BEGIN | DIVIDE | ISNULL | QTIME | TAGS | -| BETWEEN | DNODE | JOIN | QUERIES | TBNAME | -| BIGINT | DNODES | KEEP | QUERY | TIMES | -| BINARY | DOT | KEY | QUORUM | TIMESTAMP | -| BITAND | DOUBLE | KILL | RAISE | TINYINT | -| BITNOT | DROP | LE | REM | TOPIC | -| BITOR | EACH | LIKE | REPLACE | TOPICS | -| BLOCKS | END | LIMIT | REPLICA | TRIGGER | -| BOOL | EQ | LINEAR | RESET | TSERIES | -| BY | EXISTS | LOCAL | RESTRICT | UMINUS | -| CACHE | EXPLAIN | LP | ROW | UNION | -| CACHELAST | FAIL | LSHIFT | RP | UNSIGNED | -| CASCADE | FILE | LT | RSHIFT | UPDATE | -| CHANGE | FILL | MATCH | SCORES | UPLUS | -| CLUSTER | FLOAT | MAXROWS | SELECT | USE | -| COLON | FOR | MINROWS | SEMI | USER | -| COLUMN | FROM | MINUS | SESSION | USERS | -| COMMA | FSYNC | MNODES | SET | USING | -| COMP | GE | MODIFY | SHOW | VALUES | -| COMPACT | GLOB | MODULES | SLASH | VARIABLE | -| CONCAT | GRANTS | NCHAR | SLIDING | VARIABLES | -| CONFLICT | GROUP | NE | SLIMIT | VGROUPS | -| CONNECTION | GT | NONE | SMALLINT | VIEW | -| CONNECTIONS | HAVING | NOT | SOFFSET | VNODES | -| CONNS | ID | NOTNULL | STable | WAL | -| COPY | IF | NOW | STableS | WHERE | -| _C0 | _QSTART | _QSTOP | _QDURATION | _WSTART | -| _WSTOP | _WDURATION | _ROWTS | +### A + +- ABORT +- ACCOUNT +- ACCOUNTS +- ADD +- AFTER +- ALL +- ALTER +- AND +- AS +- ASC +- ATTACH + +### B + +- BEFORE +- BEGIN +- BETWEEN +- BIGINT +- BINARY +- BITAND +- BITNOT +- BITOR +- BLOCKS +- BOOL +- BY + +### C + +- CACHE +- CACHELAST +- CASCADE +- CHANGE +- CLUSTER +- COLON +- COLUMN +- COMMA +- COMP +- COMPACT +- CONCAT +- CONFLICT +- CONNECTION +- CONNECTIONS +- CONNS +- COPY +- CREATE +- CTIME + +### D + +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED +- DELETE +- DELIMITERS +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP + +### E + +- END +- EQ +- EXISTS +- EXPLAIN + +### F + +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC + +### G + +- GE +- GLOB +- GRANTS +- GROUP +- GT + +### H + +- HAVING + +### I + +- ID +- IF +- IGNORE +- IMMEDIA +- IMPORT +- IN +- INITIAL +- INSERT +- INSTEAD +- INT +- INTEGER +- INTERVA +- INTO +- IS +- ISNULL + +### J + +- JOIN + +### K + +- KEEP +- KEY +- KILL + +### L + +- LE +- LIKE +- LIMIT +- LINEAR +- LOCAL +- LP +- LSHIFT +- LT + +### M + +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES + +### N + +- NE +- NONE +- NOT +- NOTNULL +- NOW +- NULL + +### O + +- OF +- OFFSET +- OR +- ORDER + +### P + +- PARTITION +- PASS +- PLUS +- PPS +- PRECISION +- PREV +- PRIVILEGE + +### Q + +- QTIME +- QUERIE +- QUERY +- QUORUM + +### R + +- RAISE +- REM +- REPLACE +- REPLICA +- RESET +- RESTRIC +- ROW +- RP +- RSHIFT + +### S + +- SCORES +- SELECT +- SEMI +- SESSION +- SET +- SHOW +- SLASH +- SLIDING +- SLIMIT +- SMALLIN +- SOFFSET +- STable +- STableS +- STAR +- STATE +- STATEMEN +- STATE_WI +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB + +### T + +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES + +### U + +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING + +### V + +- VALUES +- VARIABLE +- VARIABLES +- VGROUPS +- VIEW +- VNODES + +### W + +- WAL +- WHERE + +### _ + +- _C0 +- _QSTART +- _QSTOP +- _QDURATION +- _WSTART +- _WSTOP +- _WDURATION ## Explanations ### TBNAME diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/12-keywords.md index 5c68e5da7e..202f223b45 100644 --- a/docs/zh/12-taos-sql/12-keywords.md +++ b/docs/zh/12-taos-sql/12-keywords.md @@ -45,48 +45,274 @@ title: TDengine 参数限制与保留关键字 目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： -| 关键字列表 | | | | | -| ----------- | ---------- | --------- | ---------- | ------------ | -| ABORT | CREATE | IGNORE | NULL | STAR | -| ACCOUNT | CTIME | IMMEDIATE | OF | STATE | -| ACCOUNTS | DATABASE | IMPORT | OFFSET | STATEMENT | -| ADD | DATABASES | IN | OR | STATE_WINDOW | -| AFTER | DAYS | INITIALLY | ORDER | STORAGE | -| ALL | DBS | INSERT | PARTITIONS | STREAM | -| ALTER | DEFERRED | INSTEAD | PASS | STREAMS | -| AND | DELIMITERS | INT | PLUS | STRING | -| AS | DESC | INTEGER | PPS | SYNCDB | -| ASC | DESCRIBE | INTERVAL | PRECISION | TABLE | -| ATTACH | DETACH | INTO | PREV | TABLES | -| BEFORE | DISTINCT | IS | PRIVILEGE | TAG | -| BEGIN | DIVIDE | ISNULL | QTIME | TAGS | -| BETWEEN | DNODE | JOIN | QUERIES | TBNAME | -| BIGINT | DNODES | KEEP | QUERY | TIMES | -| BINARY | DOT | KEY | QUORUM | TIMESTAMP | -| BITAND | DOUBLE | KILL | RAISE | TINYINT | -| BITNOT | DROP | LE | REM | TOPIC | -| BITOR | EACH | LIKE | REPLACE | TOPICS | -| BLOCKS | END | LIMIT | REPLICA | TRIGGER | -| BOOL | EQ | LINEAR | RESET | TSERIES | -| BY | EXISTS | LOCAL | RESTRICT | UMINUS | -| CACHE | EXPLAIN | LP | ROW | UNION | -| CACHELAST | FAIL | LSHIFT | RP | UNSIGNED | -| CASCADE | FILE | LT | RSHIFT | UPDATE | -| CHANGE | FILL | MATCH | SCORES | UPLUS | -| CLUSTER | FLOAT | MAXROWS | SELECT | USE | -| COLON | FOR | MINROWS | SEMI | USER | -| COLUMN | FROM | MINUS | SESSION | USERS | -| COMMA | FSYNC | MNODES | SET | USING | -| COMP | GE | MODIFY | SHOW | VALUES | -| COMPACT | GLOB | MODULES | SLASH | VARIABLE | -| CONCAT | GRANTS | NCHAR | SLIDING | VARIABLES | -| CONFLICT | GROUP | NE | SLIMIT | VGROUPS | -| CONNECTION | GT | NONE | SMALLINT | VIEW | -| CONNECTIONS | HAVING | NOT | SOFFSET | VNODES | -| CONNS | ID | NOTNULL | STABLE | WAL | -| COPY | IF | NOW | STABLES | WHERE | -| _C0 | _QSTART | _QSTOP | _QDURATION | _WSTART | -| _WSTOP | _WDURATION | _ROWTS | +### A + +- ABORT +- ACCOUNT +- ACCOUNTS +- ADD +- AFTER +- ALL +- ALTER +- AND +- AS +- ASC +- ATTACH + +### B + +- BEFORE +- BEGIN +- BETWEEN +- BIGINT +- BINARY +- BITAND +- BITNOT +- BITOR +- BLOCKS +- BOOL +- BY + +### C + +- CACHE +- CACHELAST +- CASCADE +- CHANGE +- CLUSTER +- COLON +- COLUMN +- COMMA +- COMP +- COMPACT +- CONCAT +- CONFLICT +- CONNECTION +- CONNECTIONS +- CONNS +- COPY +- CREATE +- CTIME + +### D + +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED +- DELETE +- DELIMITERS +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP + +### E + +- END +- EQ +- EXISTS +- EXPLAIN + +### F + +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC + +### G + +- GE +- GLOB +- GRANTS +- GROUP +- GT + +### H + +- HAVING + +### I + +- ID +- IF +- IGNORE +- IMMEDIA +- IMPORT +- IN +- INITIAL +- INSERT +- INSTEAD +- INT +- INTEGER +- INTERVA +- INTO +- IS +- ISNULL + +### J + +- JOIN + +### K + +- KEEP +- KEY +- KILL + +### L + +- LE +- LIKE +- LIMIT +- LINEAR +- LOCAL +- LP +- LSHIFT +- LT + +### M + +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES + +### N + +- NE +- NONE +- NOT +- NOTNULL +- NOW +- NULL + +### O + +- OF +- OFFSET +- OR +- ORDER + +### P + +- PARTITION +- PASS +- PLUS +- PPS +- PRECISION +- PREV +- PRIVILEGE + +### Q + +- QTIME +- QUERIE +- QUERY +- QUORUM + +### R + +- RAISE +- REM +- REPLACE +- REPLICA +- RESET +- RESTRIC +- ROW +- RP +- RSHIFT + +### S + +- SCORES +- SELECT +- SEMI +- SESSION +- SET +- SHOW +- SLASH +- SLIDING +- SLIMIT +- SMALLIN +- SOFFSET +- STable +- STableS +- STAR +- STATE +- STATEMEN +- STATE_WI +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB + +### T + +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES + +### U + +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING + +### V + +- VALUES +- VARIABLE +- VARIABLES +- VGROUPS +- VIEW +- VNODES + +### W + +- WAL +- WHERE + +### _ + +- _C0 +- _QSTART +- _QSTOP +- _QDURATION +- _WSTART +- _WSTOP +- _WDURATION + ## 特殊说明 ### TBNAME From fe6d46d2c0a94d236897aeb1977e81e82cc66047 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 12:41:33 +0800 Subject: [PATCH 22/25] test: disable mnode basic3 --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index db8a055362..1c694bf7fb 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -./test.sh -f tsim/mnode/basic3.sim +#./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 979420801eb0acda4cb3449b82a6332eadf9c87e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:07:27 +0800 Subject: [PATCH 23/25] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f3faa60d34..7934087ff3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1658,7 +1658,13 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + return; + } + } else { sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); } From a2039d8bb2dda45763804be5a267ebb1cd0d2693 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:16:49 +0800 Subject: [PATCH 24/25] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7934087ff3..e7f11b9a7d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1670,8 +1670,12 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); + if (gSyncEnv != NULL) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); + } else { + sError("sync env is already stop"); + } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 "", From 8dea8dbbebeca2cf8b61d739d3cadbfe7b8c6b7f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 13:40:48 +0800 Subject: [PATCH 25/25] docs: move delete data from developer guide to sql reference guide --- .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/en/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/en/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/en/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/en/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/en/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/zh/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/zh/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename docs/en/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/en/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/en/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/en/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/en/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/en/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) rename docs/zh/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/zh/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/zh/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) diff --git a/docs/en/07-develop/05-delete-data.mdx b/docs/en/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/en/07-develop/05-delete-data.mdx rename to docs/en/12-taos-sql/08-delete-data.mdx diff --git a/docs/en/12-taos-sql/07-function.md b/docs/en/12-taos-sql/10-function.md similarity index 100% rename from docs/en/12-taos-sql/07-function.md rename to docs/en/12-taos-sql/10-function.md diff --git a/docs/en/12-taos-sql/08-interval.md b/docs/en/12-taos-sql/12-interval.md similarity index 100% rename from docs/en/12-taos-sql/08-interval.md rename to docs/en/12-taos-sql/12-interval.md diff --git a/docs/en/12-taos-sql/09-limit.md b/docs/en/12-taos-sql/14-limit.md similarity index 100% rename from docs/en/12-taos-sql/09-limit.md rename to docs/en/12-taos-sql/14-limit.md diff --git a/docs/en/12-taos-sql/10-json.md b/docs/en/12-taos-sql/16-json.md similarity index 100% rename from docs/en/12-taos-sql/10-json.md rename to docs/en/12-taos-sql/16-json.md diff --git a/docs/en/12-taos-sql/11-escape.md b/docs/en/12-taos-sql/18-escape.md similarity index 100% rename from docs/en/12-taos-sql/11-escape.md rename to docs/en/12-taos-sql/18-escape.md diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/20-keywords.md similarity index 100% rename from docs/en/12-taos-sql/12-keywords.md rename to docs/en/12-taos-sql/20-keywords.md diff --git a/docs/zh/07-develop/05-delete-data.mdx b/docs/zh/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/zh/07-develop/05-delete-data.mdx rename to docs/zh/12-taos-sql/08-delete-data.mdx diff --git a/docs/zh/12-taos-sql/07-function.md b/docs/zh/12-taos-sql/10-function.md similarity index 100% rename from docs/zh/12-taos-sql/07-function.md rename to docs/zh/12-taos-sql/10-function.md diff --git a/docs/zh/12-taos-sql/08-interval.md b/docs/zh/12-taos-sql/12-interval.md similarity index 100% rename from docs/zh/12-taos-sql/08-interval.md rename to docs/zh/12-taos-sql/12-interval.md diff --git a/docs/zh/12-taos-sql/09-limit.md b/docs/zh/12-taos-sql/14-limit.md similarity index 100% rename from docs/zh/12-taos-sql/09-limit.md rename to docs/zh/12-taos-sql/14-limit.md diff --git a/docs/zh/12-taos-sql/10-json.md b/docs/zh/12-taos-sql/16-json.md similarity index 100% rename from docs/zh/12-taos-sql/10-json.md rename to docs/zh/12-taos-sql/16-json.md diff --git a/docs/zh/12-taos-sql/11-escape.md b/docs/zh/12-taos-sql/18-escape.md similarity index 100% rename from docs/zh/12-taos-sql/11-escape.md rename to docs/zh/12-taos-sql/18-escape.md diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/20-keywords.md similarity index 100% rename from docs/zh/12-taos-sql/12-keywords.md rename to docs/zh/12-taos-sql/20-keywords.md