From 87fefa9e15f638a0a950f4a05322c320995f1f44 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 01/32] [TD-14241]: add string functions --- include/libs/function/functionMgt.h | 9 +- include/libs/scalar/scalar.h | 13 + source/libs/function/src/builtins.c | 82 +++++- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/sclfunc.c | 408 ++++++++++++++++++++++++---- 5 files changed, 453 insertions(+), 61 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 7d46b543cb..0969fb203a 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -72,10 +72,15 @@ typedef enum EFunctionType { FUNCTION_TYPE_ATAN, // string function - FUNCTION_TYPE_CHAR_LENGTH = 1500, + FUNCTION_TYPE_LENGTH = 1500, + FUNCTION_TYPE_CHAR_LENGTH, FUNCTION_TYPE_CONCAT, FUNCTION_TYPE_CONCAT_WS, - FUNCTION_TYPE_LENGTH, + FUNCTION_TYPE_LOWER, + FUNCTION_TYPE_UPPER, + FUNCTION_TYPE_LTRIM, + FUNCTION_TYPE_RTRIM, + FUNCTION_TYPE_SUBSTR, // conversion function FUNCTION_TYPE_CAST = 2000, diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index c6d17ef65c..10ef0c1b23 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -42,6 +42,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut); +/* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); @@ -58,6 +59,18 @@ int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +/* String functions */ +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0a5a90d2d9..4c07ba924d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -282,6 +282,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = atanFunction, .finalizeFunc = NULL }, + { + .name = "length", + .type = FUNCTION_TYPE_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lengthFunction, + .finalizeFunc = NULL + }, + { + .name = "char_length", + .type = FUNCTION_TYPE_CHAR_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = charLengthFunction, + .finalizeFunc = NULL + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, @@ -289,7 +309,67 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = stubCheckAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = concatFunction, + .finalizeFunc = NULL + }, + { + .name = "concat_ws", + .type = FUNCTION_TYPE_CONCAT_WS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = concatWsFunction, + .finalizeFunc = NULL + }, + { + .name = "lower", + .type = FUNCTION_TYPE_LOWER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lowerFunction, + .finalizeFunc = NULL + }, + { + .name = "upper", + .type = FUNCTION_TYPE_UPPER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = upperFunction, + .finalizeFunc = NULL + }, + { + .name = "ltrim", + .type = FUNCTION_TYPE_LTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ltrimFunction, + .finalizeFunc = NULL + }, + { + .name = "rtrim", + .type = FUNCTION_TYPE_RTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = rtrimFunction, + .finalizeFunc = NULL + }, + { + .name = "substr", + .type = FUNCTION_TYPE_SUBSTR, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = substrFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index cf34fc24a9..58b62bb05e 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -47,7 +47,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); #define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) -#define GET_PARAM_BYTES(_c) ((_c)->pColumnInfoData->info.bytes) +#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) void sclFreeParam(SScalarParam *param); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index af5f07751b..15a9984c80 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -3,10 +3,14 @@ #include "sclInt.h" #include "sclvector.h" -static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { -// dst->type = src->type; -// dst->bytes = src->bytes; -// dst->num = src->num; +typedef float (*_float_fn)(float); +typedef double (*_double_fn)(double); +typedef double (*_double_fn_2)(double, double); +typedef int (*_conv_fn)(int); +typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); + +double tlog(double v, double base) { + return log(v) / log(base); } /** Math functions **/ @@ -107,14 +111,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu return TSDB_CODE_SUCCESS; } -typedef float (*_float_fn)(float); -typedef double (*_double_fn)(double); -typedef double (*_double_fn_2)(double, double); - -double tlog(double v, double base) { - return log(v) / log(base); -} - int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { @@ -216,6 +212,341 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p return TSDB_CODE_SUCCESS; } +/** String functions **/ +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char **in = (char **)pInputData->pData; + int16_t *out = (int16_t *)pOutputData->pData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + out[i] = varDataLen(in[i]); + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char **in = (char **)pInputData->pData; + int16_t *out = (int16_t *)pOutputData->pData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + if (type == TSDB_DATA_TYPE_VARCHAR) { + out[i] = varDataLen(in[i]); + } else { //NCHAR + out[i] = varDataLen(in[i]) / TSDB_NCHAR_SIZE; + } + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings + return TSDB_CODE_FAILED; + } + + SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } + pInputData[i] = pInput[i].columnData; + } + + bool hasNull = false; + for (int32_t k = 0; k < pInput->numOfRows; ++k) { + for (int32_t i = 0; i < inputNum; ++i) { + if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { + colDataSetNull_f(pOutputData->nullbitmap, k); + hasNull = true; + break; + } + } + + if (hasNull) { + continue; + } + + char *in = NULL; + char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); + + int16_t dataLen = 0; + for (int32_t i = 0; i < inputNum; ++i) { + in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); + + memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); + dataLen += varDataLen(in); + } + varDataSetLen(out, dataLen); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(pInputData); + + return TSDB_CODE_SUCCESS; +} + +int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator + return TSDB_CODE_FAILED; + } + + SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } + pInputData[i] = pInput[i].columnData; + } + + for (int32_t k = 0; k < pInput->numOfRows; ++k) { + char *sep = pInputData[0]->pData; + if (colDataIsNull_f(pInputData[0]->nullbitmap, k)) { + colDataSetNull_f(pOutputData->nullbitmap, k); + continue; + } + + char *in = NULL; + char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); + + int16_t dataLen = 0; + for (int32_t i = 1; i < inputNum; ++i) { + if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { + continue; + } + + in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); + memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); + dataLen += varDataLen(in); + + if (i < inputNum - 1) { + //insert the separator + memcpy(varDataVal(out) + dataLen, varDataVal(sep), varDataLen(sep)); + dataLen += varDataLen(sep); + } + } + varDataSetLen(out, dataLen); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(pInputData); + + return TSDB_CODE_SUCCESS; +} + +int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); + char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); + + int32_t len = varDataLen(in); + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t j = 0; j < len; ++j) { + *(varDataVal(out) + j) = convFn(*(varDataVal(in) + j)); + } + } else { //NCHAR + for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) { + *((uint32_t *)varDataVal(out) + j) = convFn(*((uint32_t *)varDataVal(in) + j)); + } + } + varDataSetLen(out, len); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + +void tltrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = 0; i < charLen; ++i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = 0; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); + } + + varDataSetLen(output, resLen); +} + +void trtrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = charLen - 1; i >= 0; --i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = charLen - 1; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + } + memcpy(varDataVal(output), varDataVal(input), resLen); + + varDataSetLen(output, resLen); +} + +int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); + char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); + + int32_t len = varDataLen(in); + int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; + trimFn(in, out, type, charLen); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + +int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 || inputNum!= 3) { + return TSDB_CODE_FAILED; + } + + int32_t subPos = 0; + GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + if (subPos == 0) { //subPos needs to be positive or negative values; + return TSDB_CODE_FAILED; + } + + int32_t subLen = INT16_MAX; + if (inputNum == 3) { + GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + if (subLen < 0) { //subLen cannot be negative + return TSDB_CODE_FAILED; + } + subLen = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subLen : subLen * TSDB_NCHAR_SIZE; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < pOutput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); + char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); + + int32_t len = varDataLen(in); + int32_t startPosBytes; + + if (subPos > 0) { + startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE; + startPosBytes = MIN(startPosBytes, len); + } else { + startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE; + startPosBytes = MAX(startPosBytes, 0); + } + + subLen = MIN(subLen, len - startPosBytes); + if (subLen > 0) { + memcpy(varDataVal(out), varDataVal(in) + startPosBytes, subLen); + } + + varDataSetLen(out, subLen); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } @@ -264,57 +595,20 @@ int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return doScalarFunction(pInput, inputNum, pOutput, roundf, round); } -static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assert(numOfInput == 1); -#if 0 - int64_t* out = (int64_t*) pOutput->data; - char* s = pLeft->data; - - for(int32_t i = 0; i < pLeft->num; ++i) { - out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); - } -#endif +int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doCaseConvFunction(pInput, inputNum, pOutput, tolower); } -static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assert(numOfInput > 0); -#if 0 - int32_t rowLen = 0; - int32_t num = 1; - for(int32_t i = 0; i < numOfInput; ++i) { - rowLen += pLeft[i].bytes; - - if (pLeft[i].num > 1) { - num = pLeft[i].num; - } - } - - pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num); - assert(pOutput->data); - - char* rstart = pOutput->data; - for(int32_t i = 0; i < num; ++i) { - - char* s = rstart; - varDataSetLen(s, 0); - for (int32_t j = 0; j < numOfInput; ++j) { - char* p1 = POINTER_SHIFT(pLeft[j].data, i * pLeft[j].bytes); - - memcpy(varDataVal(s) + varDataLen(s), varDataVal(p1), varDataLen(p1)); - varDataLen(s) += varDataLen(p1); - } - - rstart += rowLen; - } -#endif +int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doCaseConvFunction(pInput, inputNum, pOutput, toupper); } -static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - +int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doTrimFunction(pInput, inputNum, pOutput, tltrim); } -static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - +int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doTrimFunction(pInput, inputNum, pOutput, trtrim); } static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { From b0eefba1b3fa738c8a29e087c76beb0e80dce796 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 02/32] [TD-14241]: add string functions --- source/libs/scalar/src/sclfunc.c | 169 +++++++++++++++---------------- 1 file changed, 81 insertions(+), 88 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 15a9984c80..12eee76838 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -8,12 +8,13 @@ typedef double (*_double_fn)(double); typedef double (*_double_fn_2)(double, double); typedef int (*_conv_fn)(int); typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); +typedef int16_t (*_len_fn)(char *, int32_t); +/** Math functions **/ double tlog(double v, double base) { return log(v) / log(base); } -/** Math functions **/ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -213,32 +214,78 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p } /** String functions **/ -int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - int32_t type = GET_PARAM_TYPE(pInput); - if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { - return TSDB_CODE_FAILED; - } - - SColumnInfoData *pInputData = pInput->columnData; - SColumnInfoData *pOutputData = pOutput->columnData; - - char **in = (char **)pInputData->pData; - int16_t *out = (int16_t *)pOutputData->pData; - - for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); - continue; - } - - out[i] = varDataLen(in[i]); - } - - pOutput->numOfRows = pInput->numOfRows; - return TSDB_CODE_SUCCESS; +int16_t tlength(char *input, int32_t type) { + return varDataLen(input); } -int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +int16_t tcharlength(char *input, int32_t type) { + if (type == TSDB_DATA_TYPE_VARCHAR) { + return varDataLen(input); + } else { //NCHAR + return varDataLen(input) / TSDB_NCHAR_SIZE; + } +} + +void tltrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = 0; i < charLen; ++i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = 0; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); + } + + varDataSetLen(output, resLen); +} + +void trtrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = charLen - 1; i >= 0; --i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = charLen - 1; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + } + memcpy(varDataVal(output), varDataVal(input), resLen); + + varDataSetLen(output, resLen); +} + +int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { return TSDB_CODE_FAILED; @@ -256,11 +303,7 @@ int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam continue; } - if (type == TSDB_DATA_TYPE_VARCHAR) { - out[i] = varDataLen(in[i]); - } else { //NCHAR - out[i] = varDataLen(in[i]) / TSDB_NCHAR_SIZE; - } + out[i] = lenFn(in[i], type); } pOutput->numOfRows = pInput->numOfRows; @@ -403,64 +446,6 @@ int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return TSDB_CODE_SUCCESS; } -void tltrim(char *input, char *output, int32_t type, int32_t charLen) { - int32_t numOfSpaces = 0; - if (type == TSDB_DATA_TYPE_VARCHAR) { - for (int32_t i = 0; i < charLen; ++i) { - if (!isspace(*(varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } else { //NCHAR - for (int32_t i = 0; i < charLen; ++i) { - if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } - - int32_t resLen; - if (type == TSDB_DATA_TYPE_VARCHAR) { - resLen = charLen - numOfSpaces; - memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); - } else { - resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; - memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); - } - - varDataSetLen(output, resLen); -} - -void trtrim(char *input, char *output, int32_t type, int32_t charLen) { - int32_t numOfSpaces = 0; - if (type == TSDB_DATA_TYPE_VARCHAR) { - for (int32_t i = charLen - 1; i >= 0; --i) { - if (!isspace(*(varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } else { //NCHAR - for (int32_t i = charLen - 1; i < charLen; ++i) { - if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } - - int32_t resLen; - if (type == TSDB_DATA_TYPE_VARCHAR) { - resLen = charLen - numOfSpaces; - } else { - resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; - } - memcpy(varDataVal(output), varDataVal(input), resLen); - - varDataSetLen(output, resLen); -} int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { int32_t type = GET_PARAM_TYPE(pInput); @@ -611,6 +596,14 @@ int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return doTrimFunction(pInput, inputNum, pOutput, trtrim); } +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doLengthFunction(pInput, inputNum, pOutput, tlength); +} + +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doLengthFunction(pInput, inputNum, pOutput, tcharlength); +} + static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: From 89abd4e3495d2ce733a05ef59d6b17ca165fcf12 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 03/32] [TD-14241]: add string functions --- source/libs/function/src/builtins.c | 39 +++++++++++++++++++++-------- source/libs/scalar/src/sclfunc.c | 5 ++-- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4c07ba924d..a18386aff6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -478,16 +478,6 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; break; } - case FUNCTION_TYPE_CONCAT: - case FUNCTION_TYPE_ROWTS: - case FUNCTION_TYPE_TBNAME: - case FUNCTION_TYPE_QSTARTTS: - case FUNCTION_TYPE_QENDTS: - case FUNCTION_TYPE_WSTARTTS: - case FUNCTION_TYPE_WENDTS: - case FUNCTION_TYPE_WDURATION: - // todo - break; case FUNCTION_TYPE_ABS: case FUNCTION_TYPE_CEIL: @@ -512,6 +502,35 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } + case FUNCTION_TYPE_LENGTH: + case FUNCTION_TYPE_CHAR_LENGTH: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_SMALLINT].bytes, .type = TSDB_DATA_TYPE_SMALLINT }; + break; + } + + case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT_WS: + case FUNCTION_TYPE_LOWER: + case FUNCTION_TYPE_UPPER: + case FUNCTION_TYPE_LTRIM: + case FUNCTION_TYPE_RTRIM: + case FUNCTION_TYPE_SUBSTR: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + break; + } + + case FUNCTION_TYPE_ROWTS: + case FUNCTION_TYPE_TBNAME: + case FUNCTION_TYPE_QSTARTTS: + case FUNCTION_TYPE_QENDTS: + case FUNCTION_TYPE_WSTARTTS: + case FUNCTION_TYPE_WENDTS: + case FUNCTION_TYPE_WDURATION: + // todo + break; + default: ASSERT(0); // to found the fault ASAP. } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 12eee76838..8eee608ee0 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -294,7 +294,7 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char **in = (char **)pInputData->pData; + char *in = pInputData->pData; int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -303,7 +303,8 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p continue; } - out[i] = lenFn(in[i], type); + out[i] = lenFn(in, type); + in += varDataTLen(in); } pOutput->numOfRows = pInput->numOfRows; From 89aba48960cc6afd5c342cb61e15e078d6fcc074 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 04/32] [TD-14241]: concat function adoption --- source/libs/scalar/src/sclfunc.c | 39 ++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 8eee608ee0..864b8acf75 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -310,6 +310,8 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } +void allocateOutputBuf() { +} int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings @@ -318,17 +320,35 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; + char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); + char *output = NULL; + int32_t inputLen = 0; + int32_t numOfRows = pInput->numOfRows; for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { return TSDB_CODE_FAILED; } pInputData[i] = pInput[i].columnData; + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + input[i] = pInputData[i]->pData; } + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; + pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); + pOutputData->info.type = GET_PARAM_TYPE(pInput); + pOutputData->info.bytes = outputLen; + pOutputData->varmeta.length = outputLen; + pOutputData->varmeta.allocLen = outputLen; + } + output = pOutputData->pData; + bool hasNull = false; - for (int32_t k = 0; k < pInput->numOfRows; ++k) { + int32_t offset = 0; + for (int32_t k = 0; k < numOfRows; ++k) { for (int32_t i = 0; i < inputNum; ++i) { if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { colDataSetNull_f(pOutputData->nullbitmap, k); @@ -341,20 +361,21 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } - char *in = NULL; - char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); - int16_t dataLen = 0; for (int32_t i = 0; i < inputNum; ++i) { - in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); - - memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); - dataLen += varDataLen(in); + memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); + dataLen += varDataLen(input[i]); + input[i] += varDataTLen(input[i]); } - varDataSetLen(out, dataLen); + varDataSetLen(output, dataLen); + int32_t dataTLen = varDataTLen(output); + output += dataTLen; + pOutputData->varmeta.offset[k] = offset; + offset += dataTLen; } pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(input); taosMemoryFree(pInputData); return TSDB_CODE_SUCCESS; From 6f6d0e4a0997d2c592e07fefcab16d8829c6c608 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 05/32] [TD-14241]: concat function adoption --- source/libs/function/src/builtins.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a18386aff6..a19acb096d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -508,7 +508,22 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } - case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT: { + int32_t paraTypeFirst, totalBytes = 0; + for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + int32_t paraType = pParam->node.resType.type; + if (i == 0) { + paraTypeFirst = paraType; + } + if (paraType != paraTypeFirst) { + return TSDB_CODE_FAILED; + } + totalBytes += pParam->node.resType.bytes; + } + pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; + break; + } case FUNCTION_TYPE_CONCAT_WS: case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: @@ -517,7 +532,8 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_SUBSTR: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); int32_t paraType = pParam->node.resType.type; - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + //pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + pFunc->node.resType = (SDataType) { .bytes = 23, .type = paraType }; break; } From 6eb6cfe279b214bc0e457bd5f2fbd24576746255 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 06/32] [TD-14241]: concat_ws function adoption --- source/libs/function/src/builtins.c | 18 ++++++++--- source/libs/scalar/src/sclfunc.c | 47 ++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a19acb096d..4cbc91fa6a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -508,12 +508,19 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } - case FUNCTION_TYPE_CONCAT: { - int32_t paraTypeFirst, totalBytes = 0; - for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { + case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT_WS: { + int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; + int32_t firstParamIndex = 0; + if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { + firstParamIndex = 1; + SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); + sepBytes = pSep->node.resType.type; + } + for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); int32_t paraType = pParam->node.resType.type; - if (i == 0) { + if (i == firstParamIndex) { paraTypeFirst = paraType; } if (paraType != paraTypeFirst) { @@ -521,10 +528,11 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { } totalBytes += pParam->node.resType.bytes; } + //TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. + totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; break; } - case FUNCTION_TYPE_CONCAT_WS: case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: case FUNCTION_TYPE_LTRIM: diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 864b8acf75..f1eab5c05c 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -374,7 +374,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu offset += dataTLen; } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; taosMemoryFree(input); taosMemoryFree(pInputData); @@ -388,45 +388,70 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; + char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); + char *output = NULL; + int32_t inputLen = 0; + int32_t numOfRows = pInput[1].numOfRows; for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { return TSDB_CODE_FAILED; } pInputData[i] = pInput[i].columnData; + if (i == 0) { + // calculate required separator space + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * (inputNum - 2); + } else { + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + } + input[i] = pInputData[i]->pData; } - for (int32_t k = 0; k < pInput->numOfRows; ++k) { + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; + pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); + pOutputData->info.type = GET_PARAM_TYPE(pInput); + pOutputData->info.bytes = outputLen; + pOutputData->varmeta.length = outputLen; + pOutputData->varmeta.allocLen = outputLen; + } + output = pOutputData->pData; + + int32_t offset = 0; + for (int32_t k = 0; k < numOfRows; ++k) { char *sep = pInputData[0]->pData; if (colDataIsNull_f(pInputData[0]->nullbitmap, k)) { colDataSetNull_f(pOutputData->nullbitmap, k); continue; } - char *in = NULL; - char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); - int16_t dataLen = 0; for (int32_t i = 1; i < inputNum; ++i) { if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { continue; } - in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); - memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); - dataLen += varDataLen(in); + memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); + dataLen += varDataLen(input[i]); + input[i] += varDataTLen(input[i]); if (i < inputNum - 1) { //insert the separator - memcpy(varDataVal(out) + dataLen, varDataVal(sep), varDataLen(sep)); + memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep)); dataLen += varDataLen(sep); } } - varDataSetLen(out, dataLen); + varDataSetLen(output, dataLen); + int32_t dataTLen = varDataTLen(output); + output += dataTLen; + pOutputData->varmeta.offset[k] = offset; + offset += dataTLen; } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; + taosMemoryFree(input); taosMemoryFree(pInputData); return TSDB_CODE_SUCCESS; From dddf621aa7a4504f756c1d2aa96a18a417b8a85c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 07/32] [TD-14241]: concat/concat_ws fix input has both constant and column issue --- source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 52 +++++++++++++++++++---------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4cbc91fa6a..957e7ee406 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -526,6 +526,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { if (paraType != paraTypeFirst) { return TSDB_CODE_FAILED; } + //TODO: for constants also needs numOfRows totalBytes += pParam->node.resType.bytes; } //TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index f1eab5c05c..21d8aee0ec 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -310,7 +310,13 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } -void allocateOutputBuf() { + +static void setVarTypeOutputBuf(SColumnInfoData *pOutputData, int32_t len, int32_t type) { + pOutputData->pData = taosMemoryCalloc(len, sizeof(char)); + pOutputData->info.type = type; + pOutputData->info.bytes = len; + pOutputData->varmeta.length = len; + pOutputData->varmeta.allocLen = len; } int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { @@ -324,25 +330,30 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu char *output = NULL; int32_t inputLen = 0; - int32_t numOfRows = pInput->numOfRows; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; + } + } for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { return TSDB_CODE_FAILED; } pInputData[i] = pInput[i].columnData; - inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; input[i] = pInputData[i]->pData; + if (pInput[i].numOfRows == 1) { + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; + } else { + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + } } //allocate output buf if (pOutputData->pData == NULL) { int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; - pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); - pOutputData->info.type = GET_PARAM_TYPE(pInput); - pOutputData->info.bytes = outputLen; - pOutputData->varmeta.length = outputLen; - pOutputData->varmeta.allocLen = outputLen; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); } output = pOutputData->pData; @@ -365,7 +376,9 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu for (int32_t i = 0; i < inputNum; ++i) { memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); dataLen += varDataLen(input[i]); - input[i] += varDataTLen(input[i]); + if (pInput[i].numOfRows != 1) { + input[i] += varDataTLen(input[i]); + } } varDataSetLen(output, dataLen); int32_t dataTLen = varDataTLen(output); @@ -392,7 +405,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *output = NULL; int32_t inputLen = 0; - int32_t numOfRows = pInput[1].numOfRows; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; + } + } for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { @@ -401,7 +419,9 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pInputData[i] = pInput[i].columnData; if (i == 0) { // calculate required separator space - inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * (inputNum - 2); + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2); + } else if (pInput[i].numOfRows == 1) { + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; } else { inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; } @@ -411,11 +431,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p //allocate output buf if (pOutputData->pData == NULL) { int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; - pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); - pOutputData->info.type = GET_PARAM_TYPE(pInput); - pOutputData->info.bytes = outputLen; - pOutputData->varmeta.length = outputLen; - pOutputData->varmeta.allocLen = outputLen; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(&pInput[1])); } output = pOutputData->pData; @@ -435,7 +451,9 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); dataLen += varDataLen(input[i]); - input[i] += varDataTLen(input[i]); + if (pInput[i].numOfRows != 1) { + input[i] += varDataTLen(input[i]); + } if (i < inputNum - 1) { //insert the separator From c846c6ebf30ba307e5ba028253294a6637de0b25 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 08/32] [TD-14241]: lower/upper function adoption --- source/libs/function/src/builtins.c | 6 +++--- source/libs/scalar/src/sclfunc.c | 26 +++++++++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 957e7ee406..dd9e32655f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -540,9 +540,9 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_RTRIM: case FUNCTION_TYPE_SUBSTR: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); - int32_t paraType = pParam->node.resType.type; - //pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; - pFunc->node.resType = (SDataType) { .bytes = 23, .type = paraType }; + int32_t paraType = pParam->node.resType.type; + int32_t paraBytes = pParam->node.resType.bytes; + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; break; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 21d8aee0ec..e6eb3c43b4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -484,26 +484,38 @@ int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = pInputData->varmeta.length; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); + } + + char *input = pInputData->pData; + char *output = pOutputData->pData; + + int32_t offset = 0; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_f(pInputData->nullbitmap, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } - char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); - char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); - - int32_t len = varDataLen(in); + int32_t len = varDataLen(input); if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t j = 0; j < len; ++j) { - *(varDataVal(out) + j) = convFn(*(varDataVal(in) + j)); + *(varDataVal(output) + j) = convFn(*(varDataVal(input) + j)); } } else { //NCHAR for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) { - *((uint32_t *)varDataVal(out) + j) = convFn(*((uint32_t *)varDataVal(in) + j)); + *((uint32_t *)varDataVal(output) + j) = convFn(*((uint32_t *)varDataVal(input) + j)); } } - varDataSetLen(out, len); + varDataSetLen(output, len); + input += varDataTLen(input); + output += varDataTLen(output); + + pOutputData->varmeta.offset[i] = offset; + offset += len + VARSTR_HEADER_SIZE; } pOutput->numOfRows = pInput->numOfRows; From 64184611d7e35c1aa84e59a91640e6b380e6b811 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 09/32] [TD-14241]: ltrim/rtrim function adoption --- source/libs/scalar/src/sclfunc.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e6eb3c43b4..9f481c8fb2 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -533,18 +533,32 @@ int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = pInputData->varmeta.length; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); + } + + char *input = pInputData->pData; + char *output = pOutputData->pData; + + int32_t offset = 0; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_f(pInputData->nullbitmap, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } - char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); - char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); - - int32_t len = varDataLen(in); + int32_t len = varDataLen(input); int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; - trimFn(in, out, type, charLen); + trimFn(input, output, type, charLen); + + varDataSetLen(output, len); + input += varDataTLen(input); + output += varDataTLen(output); + + pOutputData->varmeta.offset[i] = offset; + offset += len + VARSTR_HEADER_SIZE; } pOutput->numOfRows = pInput->numOfRows; From 5a1548bf1548b9a9f66d8ea7283037703482ae13 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 10/32] [TD-14241]: substr function adoption --- source/libs/function/src/builtins.c | 49 ++++++++++++++++------------- source/libs/scalar/src/sclfunc.c | 30 ++++++++++++------ 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index dd9e32655f..dc43a3508f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -510,29 +510,36 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_CONCAT: case FUNCTION_TYPE_CONCAT_WS: { - int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; - int32_t firstParamIndex = 0; - if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { - firstParamIndex = 1; - SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); - sepBytes = pSep->node.resType.type; - } - for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { + int32_t paraType, paraBytes = 0; + for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); - int32_t paraType = pParam->node.resType.type; - if (i == firstParamIndex) { - paraTypeFirst = paraType; - } - if (paraType != paraTypeFirst) { - return TSDB_CODE_FAILED; - } - //TODO: for constants also needs numOfRows - totalBytes += pParam->node.resType.bytes; + paraBytes += pParam->node.resType.bytes; + paraType = pParam->node.resType.type; } - //TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. - totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; - pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; - break; + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; + //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; + //int32_t firstParamIndex = 0; + //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { + // firstParamIndex = 1; + // SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); + // sepBytes = pSep->node.resType.type; + //} + //for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { + // SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + // int32_t paraType = pParam->node.resType.type; + // if (i == firstParamIndex) { + // paraTypeFirst = paraType; + // } + // if (paraType != paraTypeFirst) { + // return TSDB_CODE_FAILED; + // } + // //TODO: for constants also needs numOfRows + // totalBytes += pParam->node.resType.bytes; + //} + ////TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. + //totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; + //pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; + //break; } case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 9f481c8fb2..222f7ac69a 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -350,7 +350,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } } - //allocate output buf + allocate output buf if (pOutputData->pData == NULL) { int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); @@ -567,7 +567,7 @@ int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 2 || inputNum!= 3) { + if (inputNum != 2 && inputNum!= 3) { return TSDB_CODE_FAILED; } @@ -589,16 +589,23 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - for (int32_t i = 0; i < pOutput->numOfRows; ++i) { + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = pInputData->varmeta.length; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); + } + + char *input = pInputData->pData; + char *output = pOutputData->pData; + + int32_t offset = 0; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_f(pInputData->nullbitmap, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } - char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); - char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); - - int32_t len = varDataLen(in); + int32_t len = varDataLen(input); int32_t startPosBytes; if (subPos > 0) { @@ -611,10 +618,15 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu subLen = MIN(subLen, len - startPosBytes); if (subLen > 0) { - memcpy(varDataVal(out), varDataVal(in) + startPosBytes, subLen); + memcpy(varDataVal(output), varDataVal(input) + startPosBytes, subLen); } - varDataSetLen(out, subLen); + varDataSetLen(output, subLen); + input += varDataTLen(input); + output += varDataTLen(output); + + pOutputData->varmeta.offset[i] = offset; + offset += subLen + VARSTR_HEADER_SIZE; } pOutput->numOfRows = pInput->numOfRows; From dfac724697c05146b007a48f8dd21f98f490e52f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 16:33:17 +0800 Subject: [PATCH 11/32] handle except --- source/libs/index/src/indexFst.c | 9 +++++++++ source/libs/index/test/fstTest.cc | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index a146564c9e..bc3ecea7a5 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -642,6 +642,9 @@ static const char* fstNodeState(FstNode* node) { } void fstNodeDestroy(FstNode* node) { + if (node == NULL) { + return; + } fstSliceDestroy(&node->data); taosMemoryFree(node); } @@ -1247,7 +1250,10 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); + StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; + node = NULL; + taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); @@ -1271,6 +1277,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { return true; } } + + fstNodeDestroy(node); + uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { StreamState* s = taosArrayGet(sws->stack, sz - 1); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index c45f655746..bfe1bb21bf 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -480,6 +480,15 @@ void checkFstCheckIteratorRange2() { assert(result.size() == 4); automCtxDestroy(ctx); } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "bb", GE, "ed", LT, result); + assert(result.size() == 3); + automCtxDestroy(ctx); + } { // range search std::vector result; @@ -635,11 +644,11 @@ int main(int argc, char* argv[]) { // path suid colName ver // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); //} - // checkFstCheckIterator1(); - // checkFstCheckIterator2(); - // checkFstCheckIteratorPrefix(); - // checkFstCheckIteratorRange1(); - // checkFstCheckIteratorRange2(); + checkFstCheckIterator1(); + checkFstCheckIterator2(); + checkFstCheckIteratorPrefix(); + checkFstCheckIteratorRange1(); + checkFstCheckIteratorRange2(); checkFstCheckIteratorRange3(); // checkFstLongTerm(); // checkFstPrefixSearch(); From 20f1e3f5e516fde2f0f2a388498da1ff5b219907 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 17:22:19 +0800 Subject: [PATCH 12/32] refact dnode --- include/libs/transport/trpc.h | 7 +++++-- source/dnode/mgmt/bm/src/bmWorker.c | 2 +- source/dnode/mgmt/main/inc/dnd.h | 2 -- source/dnode/mgmt/main/src/dndExec.c | 2 +- source/dnode/mgmt/main/src/dndMsg.c | 2 +- source/dnode/mgmt/main/src/dndTransport.c | 25 ++++++++++------------- source/dnode/mgmt/mm/src/mmWorker.c | 4 ++-- source/dnode/mgmt/qm/src/qmWorker.c | 2 +- source/dnode/mgmt/vm/src/vmWorker.c | 4 ++-- 9 files changed, 24 insertions(+), 26 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 157e0cb721..3e2f596784 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -58,6 +58,9 @@ typedef struct { void *pNode; } SNodeMsg; +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); +typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + typedef struct SRpcInit { uint16_t localPort; // local port char * label; // for debug purpose @@ -74,10 +77,10 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + RpcCfp cfp; // call back to retrieve the client auth info, for server app only - int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + RpcAfp afp;; void *parent; } SRpcInit; diff --git a/source/dnode/mgmt/bm/src/bmWorker.c b/source/dnode/mgmt/bm/src/bmWorker.c index 932c008e34..a5a97f6af0 100644 --- a/source/dnode/mgmt/bm/src/bmWorker.c +++ b/source/dnode/mgmt/bm/src/bmWorker.c @@ -18,7 +18,7 @@ static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rpcRsp); + tmsgSendRsp(&rpcRsp); dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index f3a409f257..cf33787108 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -166,8 +166,6 @@ int32_t dndInitClient(SDnode *pDnode); void dndCleanupClient(SDnode *pDnode); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); #ifdef __cplusplus diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index 8837da6d48..3bbdb1a686 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -78,7 +78,7 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); if (pRpc->msgType & 1U) { SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } dTrace("msg:%p, is freed", pMsg); diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index bb0f8ccb89..76b66467bf 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -86,7 +86,7 @@ _OVER: dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); if (pRpc->msgType & 1U) { SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } dTrace("msg:%p, is freed", pMsg); taosFreeQitem(pMsg); diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index d463f2ba7a..f863db2a0c 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -36,8 +36,7 @@ static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, S dndProcessRpcMsg(pWrapper, pMsg, pEpSet); } -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; +static void dndProcessResponse(SDnode *pDnode, SRpcMsg *pRsp, SEpSet *pEpSet) { STransMgmt *pMgmt = &pDnode->trans; tmsg_t msgType = pRsp->msgType; @@ -69,7 +68,7 @@ int32_t dndInitClient(SDnode *pDnode) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND"; rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; + rpcInit.cfp = (RpcCfp)dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; @@ -101,8 +100,7 @@ void dndCleanupClient(SDnode *pDnode) { } } -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; +static void dndProcessRequest(SDnode *pDnode, SRpcMsg *pReq, SEpSet *pEpSet) { STransMgmt *pMgmt = &pDnode->trans; tmsg_t msgType = pReq->msgType; @@ -179,10 +177,8 @@ static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *e return code; } -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { +static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (dndGetHideUserAuth(pDnode, user, spi, encrypt, secret, ckey) == 0) { dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } @@ -244,11 +240,11 @@ int32_t dndInitServer(SDnode *pDnode) { rpcInit.localPort = pDnode->serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; + rpcInit.cfp = (RpcCfp)dndProcessRequest; rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; + rpcInit.afp = (RpcAfp)dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; pMgmt->serverRpc = rpcOpen(&rpcInit); @@ -363,6 +359,7 @@ int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + memcpy(pHead, pReq, sizeof(SRpcMsg)); memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); @@ -372,7 +369,7 @@ int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) } } -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { +static void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { if (pWrapper->procType != PROC_CHILD) { dndSendRpcRsp(pWrapper, pRsp); } else { @@ -380,7 +377,7 @@ void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } } -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { +static void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != PROC_CHILD) { rpcRegisterBrokenLinkArg(pMsg); } else { @@ -406,4 +403,4 @@ SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { .releaseHandleFp = dndReleaseHandle, }; return msgCb; -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/mm/src/mmWorker.c b/source/dnode/mgmt/mm/src/mmWorker.c index c4aafe05e4..cde4eb853a 100644 --- a/source/dnode/mgmt/mm/src/mmWorker.c +++ b/source/dnode/mgmt/mm/src/mmWorker.c @@ -37,7 +37,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); } SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } @@ -60,7 +60,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { if (pRpc->handle != NULL && code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } diff --git a/source/dnode/mgmt/qm/src/qmWorker.c b/source/dnode/mgmt/qm/src/qmWorker.c index 14efb311b1..5ceb9dbf88 100644 --- a/source/dnode/mgmt/qm/src/qmWorker.c +++ b/source/dnode/mgmt/qm/src/qmWorker.c @@ -18,7 +18,7 @@ static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { diff --git a/source/dnode/mgmt/vm/src/vmWorker.c b/source/dnode/mgmt/vm/src/vmWorker.c index 4be6311cf8..c228e6e7dd 100644 --- a/source/dnode/mgmt/vm/src/vmWorker.c +++ b/source/dnode/mgmt/vm/src/vmWorker.c @@ -18,7 +18,7 @@ static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { @@ -116,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; - dndSendRsp(pVnode->pWrapper, pRsp); + tmsgSendRsp(pRsp); taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; From a651affa47a71a3e1befd9b75107f251ed7d5e43 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 2 Apr 2022 17:47:27 +0800 Subject: [PATCH 13/32] [TD-13756]: log file open fail error. --- source/util/src/tlog.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1f26ab7c51..563acfec78 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -214,6 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; + taosSsleep(3); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); @@ -347,16 +348,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (pFile == NULL) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - TdFilePtr pOldFile = tsLogObj.logHandle->pFile; - tsLogObj.logHandle->pFile = pFile; - taosUnLockLogFile(pOldFile); - taosCloseFile(&pOldFile); taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines From eb18ad240b45a34d6e06714c6644e7e4f8f36d7e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 2 Apr 2022 17:54:29 +0800 Subject: [PATCH 14/32] [TD-13756]: log file open fail error. --- source/os/src/osFile.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 1b73f96896..a7855539a4 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t nleft = count; int64_t nwritten = 0; @@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -429,7 +429,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); @@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif @@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif @@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return ftruncate(pFile->fd, l_size); #endif @@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { if (pFile == NULL) { return NULL; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); return ptr; From 2eb30ee0f0a6e486a6b3c87471b2e4d7ca4f8b8c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 18:08:38 +0800 Subject: [PATCH 15/32] refact transport --- source/dnode/mgmt/dm/inc/dmInt.h | 1 + source/dnode/mgmt/dm/src/dmInt.c | 14 +-- source/dnode/mgmt/dm/src/dmMsg.c | 4 +- source/dnode/mgmt/main/inc/dnd.h | 12 +-- source/dnode/mgmt/main/inc/dndInt.h | 1 - source/dnode/mgmt/main/src/dndExec.c | 63 ----------- source/dnode/mgmt/main/src/dndInt.c | 1 + source/dnode/mgmt/main/src/dndTransport.c | 125 ++++++++++++++++------ 8 files changed, 106 insertions(+), 115 deletions(-) diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index 3036d1f5ad..79d7e926ce 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -33,6 +33,7 @@ typedef struct SDnodeMgmt { SRWLatch latch; SSingleWorker mgmtWorker; SSingleWorker statusWorker; + SMsgCb msgCb; const char *path; SDnode *pDnode; SMgmtWrapper *pWrapper; diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index 3c5f394d5e..8c46c402ab 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -112,17 +112,14 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return -1; } - if (dndInitServer(pDnode) != 0) { - dError("failed to init trans server since %s", terrstr()); - return -1; - } - - if (dndInitClient(pDnode) != 0) { - dError("failed to init trans client since %s", terrstr()); + if (dndInitTrans(pDnode) != 0) { + dError("failed to init transport since %s", terrstr()); return -1; } pWrapper->pMgmt = pMgmt; + pMgmt->msgCb = dndCreateMsgcb(pWrapper); + dInfo("dnode-mgmt is initialized"); return 0; } @@ -151,8 +148,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); + dndCleanupTrans(pDnode); dInfo("dnode-mgmt is cleaned up"); } diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index 3795cffbfa..02a288e2fd 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -57,7 +57,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { pMgmt->statusSent = 1; dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); - dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + tmsgSendReq(&pMgmt->msgCb, &epSet, &rpcMsg); } static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index cf33787108..d948aa305c 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -92,6 +92,7 @@ typedef struct SMgmtWrapper { char *path; int32_t refCount; SRWLatch latch; + ENodeType ntype; bool deployed; bool required; EProcType procType; @@ -160,13 +161,10 @@ const char *dndNodeProcStr(ENodeType ntype); const char *dndEventStr(EDndEvent ev); // dndTransport.h -int32_t dndInitServer(SDnode *pDnode); -void dndCleanupServer(SDnode *pDnode); -int32_t dndInitClient(SDnode *pDnode); -void dndCleanupClient(SDnode *pDnode); -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); -SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +int32_t dndInitTrans(SDnode *pDnode); +void dndCleanupTrans(SDnode *pDnode); +SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 0c4ada1df3..b4d9de5184 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -49,7 +49,6 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndTransport.c int32_t dndInitMsgHandle(SDnode *pDnode); -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); // dndFile.c TdFilePtr dndCheckRunning(const char *dataDir); diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index 3bbdb1a686..a98c0954fe 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -65,53 +65,6 @@ void dndCloseNode(SMgmtWrapper *pWrapper) { dDebug("node:%s, mgmt has been closed", pWrapper->name); } -static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - SRpcMsg *pRpc = &pMsg->rpcMsg; - pRpc->pCont = pCont; - dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); - - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - int32_t code = (*msgFp)(pWrapper, pMsg); - - if (code != 0) { - dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - tmsgSendRsp(&rsp); - } - - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pCont); - } -} - -static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - pMsg->pCont = pCont; - dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); - - switch (ftype) { - case PROC_REGIST: - rpcRegisterBrokenLinkArg(pMsg); - break; - case PROC_RELEASE: - rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); - rpcFreeCont(pCont); - break; - case PROC_REQ: - dndSendReqToMnode(pWrapper, pMsg); - // dndSendReq(pWrapper, (const SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); - break; - case PROC_RSP: - dndSendRpcRsp(pWrapper, pMsg); - break; - default: - break; - } - taosMemoryFree(pMsg); -} static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { char tstr[8] = {0}; @@ -135,22 +88,6 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { return 0; } -static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .shm = pWrapper->shm, - .pParent = pWrapper, - .name = pWrapper->name}; - return cfg; -} static int32_t dndRunInSingleProcess(SDnode *pDnode) { dInfo("dnode run in single process"); diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 25e4d89c5b..5e4b0a1f7b 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -95,6 +95,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { pWrapper->path = strdup(path); pWrapper->shm.id = -1; pWrapper->pDnode = pDnode; + pWrapper->ntype = n; if (pWrapper->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index f863db2a0c..cefa77890d 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -61,7 +61,7 @@ static void dndProcessResponse(SDnode *pDnode, SRpcMsg *pRsp, SEpSet *pEpSet) { } } -int32_t dndInitClient(SDnode *pDnode) { +static int32_t dndInitClient(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; SRpcInit rpcInit; @@ -91,7 +91,7 @@ int32_t dndInitClient(SDnode *pDnode) { return 0; } -void dndCleanupClient(SDnode *pDnode) { +static void dndCleanupClient(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; if (pMgmt->clientRpc) { rpcClose(pMgmt->clientRpc); @@ -227,7 +227,7 @@ static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, ch return rpcRsp.code; } -int32_t dndInitServer(SDnode *pDnode) { +static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); @@ -257,7 +257,7 @@ int32_t dndInitServer(SDnode *pDnode) { return 0; } -void dndCleanupServer(SDnode *pDnode) { +static void dndCleanupServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; if (pMgmt->serverRpc) { rpcClose(pMgmt->serverRpc); @@ -266,6 +266,17 @@ void dndCleanupServer(SDnode *pDnode) { } } +int32_t dndInitTrans(SDnode *pDnode) { + if (dndInitServer(pDnode) != 0) return -1; + if (dndInitClient(pDnode) != 0) return -1; + return 0; +} + +void dndCleanupTrans(SDnode *pDnode) { + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); +} + int32_t dndInitMsgHandle(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; @@ -315,46 +326,28 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *p return 0; } -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - STransMgmt *pTrans = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper2 = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper2 != NULL) { - dmGetMnodeEpSet(pWrapper2->pMgmt, &epSet); - dndReleaseWrapper(pWrapper2); - } - return dndSendRpcReq(pTrans, &epSet, pReq); -} - -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { +static void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { if (pRsp->code == TSDB_CODE_APP_NOT_READY) { - SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); - if (pDnodeWrapper != NULL) { - dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); - dndReleaseWrapper(pDnodeWrapper); - } else { - rpcSendResponse(pRsp); + if (pWrapper->ntype == MNODE) { + dmSendRedirectRsp(pWrapper->pMgmt, pRsp); + return; } - } else { - rpcSendResponse(pRsp); } + + rpcSendResponse(pRsp); } -int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { +static int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { + if (dndGetStatus(pWrapper->pDnode) != DND_STAT_RUNNING) { terrno = TSDB_CODE_DND_OFFLINE; dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); return -1; } if (pWrapper->procType != PROC_CHILD) { - return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); + return dndSendRpcReq(&pWrapper->pDnode->trans, pEpSet, pReq); } else { - int32_t headLen = sizeof(SRpcMsg) + sizeof(SEpSet); - char *pHead = taosMemoryMalloc(headLen); + char *pHead = taosMemoryMalloc(sizeof(SRpcMsg) + sizeof(SEpSet)); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -362,8 +355,8 @@ int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) memcpy(pHead, pReq, sizeof(SRpcMsg)); memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); - - taosProcPutToParentQ(pWrapper->pProc, pReq, headLen, pReq->pCont, pReq->contLen, PROC_REQ); + taosProcPutToParentQ(pWrapper->pProc, pHead, sizeof(SRpcMsg) + sizeof(SEpSet), pReq->pCont, pReq->contLen, + PROC_REQ); taosMemoryFree(pHead); return 0; } @@ -404,3 +397,67 @@ SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { }; return msgCb; } + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + pMsg->pCont = pCont; + dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); + + switch (ftype) { + case PROC_REGIST: + rpcRegisterBrokenLinkArg(pMsg); + break; + case PROC_RELEASE: + rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); + rpcFreeCont(pCont); + break; + case PROC_REQ: + dndSendRpcReq(&pWrapper->pDnode->trans, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); + break; + case PROC_RSP: + dndSendRpcRsp(pWrapper, pMsg); + break; + default: + break; + } + taosMemoryFree(pMsg); +} + +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { + SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .shm = pWrapper->shm, + .pParent = pWrapper, + .name = pWrapper->name}; + return cfg; +} \ No newline at end of file From 781e869e4e716f72503665cc37bc3a78d20f24ae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 18:11:45 +0800 Subject: [PATCH 16/32] handle except --- source/libs/transport/test/rclient.c | 45 +++++++++++++++------------- source/libs/transport/test/rserver.c | 7 ++++- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 7d3c3aa012..cdf0908167 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -23,20 +23,20 @@ #include "tutil.h" typedef struct { - int index; - SEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t * pOverSem; + int index; + SEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t * pOverSem; TdThread thread; - void * pRpc; + void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); + // tError("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + // pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -51,9 +51,9 @@ static void *sendRequest(void *param) { SInfo * pInfo = (SInfo *)param; SRpcMsg rpcMsg = {0}; - tDebug("thread:%d, start to send request", pInfo->index); + tError("thread:%d, start to send request", pInfo->index); - tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + tError("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); int u100 = 0; int u500 = 0; int u1000 = 0; @@ -68,7 +68,7 @@ static void *sendRequest(void *param) { // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + if (pInfo->num % 20000 == 0) tError("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); int64_t end = taosGetTimestampUs() - start; @@ -88,7 +88,7 @@ static void *sendRequest(void *param) { } tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); - tDebug("thread:%d, it is over", pInfo->index); + tError("thread:%d, it is over", pInfo->index); tcount++; return NULL; @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - TdThreadAttr thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -124,7 +124,7 @@ int main(int argc, char *argv[]) { rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.connType = TAOS_CONN_CLIENT; - rpcDebugFlag = 143; + rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -170,6 +170,10 @@ int main(int argc, char *argv[]) { } } + const char *path = "/tmp/transport/client"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); @@ -178,8 +182,8 @@ int main(int argc, char *argv[]) { return -1; } - tInfo("client is initialized"); - tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); + tError("client is initialized"); + tError("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; @@ -208,8 +212,9 @@ int main(int argc, char *argv[]) { endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; float usedTime = (endTime - startTime) / 1000.0f; // mseconds - tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); + tError("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tError("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, + msgSize); int ch = getchar(); UNUSED(ch); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 3a086371b0..8ed3bbc960 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -125,7 +125,7 @@ int main(int argc, char *argv[]) { rpcInit.idleTime = 2 * 1500; rpcInit.afp = retrieveAuthInfo; - rpcDebugFlag = 143; + rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -160,6 +160,11 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; + + const char *path = "/tmp/transport/server"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); From 1cbdec1aba78a5740e149b661e6d5386e706730a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 2 Apr 2022 18:25:57 +0800 Subject: [PATCH 17/32] [td-14393] fix bug, update sim script. --- include/common/tcommon.h | 6 +- source/client/src/clientMain.c | 17 +++- tests/script/tsim/query/session.sim | 148 ++++++++++++++-------------- 3 files changed, 92 insertions(+), 79 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 45f608b24e..f2fec58d8f 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -53,12 +53,12 @@ typedef struct { typedef struct SColumnDataAgg { int16_t colId; - int64_t sum; - int64_t max; - int64_t min; int16_t maxIndex; int16_t minIndex; int16_t numOfNull; + int64_t sum; + int64_t max; + int64_t min; } SColumnDataAgg; typedef struct SDataBlockInfo { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0ca6169883..ee4fc7fcb5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -264,8 +264,17 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - // TODO - return NULL; + SRequestObj* pRequest = (SRequestObj*) res; + if (pRequest == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + if (taos_is_update_query(res)) { + return NULL; + } + + return &pRequest->body.resInfo.row; } // todo intergrate with tDataTypes @@ -313,7 +322,9 @@ int taos_affected_rows(TAOS_RES *res) { return pResInfo->numOfRows; } -int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } +int taos_result_precision(TAOS_RES *res) { + return TSDB_TIME_PRECISION_MILLI; +} int taos_select_db(TAOS *taos, const char *db) { STscObj *pObj = (STscObj *)taos; diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index 98f63f42e3..635e6a60d7 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -79,7 +79,7 @@ loop_test_pos: # session(ts,5a) print ====> select count(*) from dev_001 session(ts,5a) -sql select count(*) from dev_001 session(ts,5a) +sql select _wstartts, count(*) from dev_001 session(ts,5a) print ====> rows: $rows print ====> $data00 $data01 $data02 $data03 $data04 $data05 print ====> $data10 $data11 $data12 $data13 $data14 $data15 @@ -100,7 +100,7 @@ endi print ====> select count(*) from (select * from dev_001) session(ts,5a) -sql select count(*) from (select * from dev_001) session(ts,5a) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,5a) if $rows != 15 then return -1 endi @@ -109,7 +109,7 @@ if $data01 != 2 then endi print ====> select count(*) from dev_001 session(ts,1s) -sql select count(*) from dev_001 session(ts,1s) +sql select _wstartts, count(*) from dev_001 session(ts,1s) if $rows != 12 then return -1 endi @@ -118,7 +118,7 @@ if $data01 != 5 then endi print ====> select count(*) from (select * from dev_001) session(ts,1s) -sql select count(*) from (select * from dev_001) session(ts,1s) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1s) if $rows != 12 then return -1 endi @@ -127,7 +127,7 @@ if $data01 != 5 then endi print ====> select count(*) from dev_001 session(ts,1000a) -sql select count(*) from dev_001 session(ts,1000a) +sql select _wstartts, count(*) from dev_001 session(ts,1000a) if $rows != 12 then return -1 endi @@ -136,7 +136,7 @@ if $data01 != 5 then endi print ====> select count(*) from (select * from dev_001) session(ts,1000a) -sql select count(*) from (select * from dev_001) session(ts,1000a) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1000a) if $rows != 12 then return -1 endi @@ -145,7 +145,7 @@ if $data01 != 5 then endi print ====> select count(*) from dev_001 session(ts,1m) -sql select count(*) from dev_001 session(ts,1m) +sql select _wstartts, count(*) from dev_001 session(ts,1m) if $rows != 9 then return -1 endi @@ -154,7 +154,7 @@ if $data01 != 8 then endi print ====> select count(*) from (select * from dev_001) session(ts,1m) -sql select count(*) from (select * from dev_001) session(ts,1m) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1m) if $rows != 9 then return -1 endi @@ -163,7 +163,7 @@ if $data01 != 8 then endi print ====> select count(*) from dev_001 session(ts,1h) -sql select count(*) from dev_001 session(ts,1h) +sql select _wstartts, count(*) from dev_001 session(ts,1h) if $rows != 6 then return -1 endi @@ -172,7 +172,7 @@ if $data01 != 11 then endi print ====> select count(*) from (select * from dev_001) session(ts,1h) -sql select count(*) from (select * from dev_001) session(ts,1h) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1h) if $rows != 6 then return -1 endi @@ -181,7 +181,7 @@ if $data01 != 11 then endi print ====> select count(*) from dev_001 session(ts,1d) -sql select count(*) from dev_001 session(ts,1d) +sql select _wstartts, count(*) from dev_001 session(ts,1d) if $rows != 4 then return -1 endi @@ -190,7 +190,7 @@ if $data01 != 13 then endi print ====> select count(*) from (select * from dev_001) session(ts,1d) -sql select count(*) from (select * from dev_001) session(ts,1d) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1d) if $rows != 4 then return -1 endi @@ -199,7 +199,7 @@ if $data01 != 13 then endi print ====> select count(*) from dev_001 session(ts,1w) -sql select count(*) from dev_001 session(ts,1w) +sql select _wstartts, count(*) from dev_001 session(ts,1w) if $rows != 2 then return -1 endi @@ -208,7 +208,7 @@ if $data01 != 15 then endi print ====> select count(*) from (select * from dev_001) session(ts,1w) -sql select count(*) from (select * from dev_001) session(ts,1w) +sql select _wstartts, count(*) from (select * from dev_001) session(ts,1w) if $rows != 2 then return -1 endi @@ -216,71 +216,73 @@ if $data01 != 15 then return -1 endi + +print ====> leastsquares not supported yet. print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) -sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) -if $rows != 2 then - return -1 -endi -if $data01 != 13 then - return -1 -endi -if $data02 != 1 then - return -1 -endi -if $data03 != 13 then - return -1 -endi -if $data04 != 7 then - return -1 -endi -if $data05 != 91 then - return -1 -endi -if $data06 != 1 then - return -1 -endi -if $data07 != 13 then - return -1 -endi -if $data08 != @{slop:1.000000, intercept:0.000000}@ then - return -1 -endi -if $data09 != 12 then - return -1 -endi +#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1),spread(tagtype),stddev(tagtype),percentile(tagtype,0) from dev_001 where ts <'2020-05-20 0:0:0' session(ts,1d) +#if $rows != 2 then +# return -1 +#endi +#if $data01 != 13 then +# return -1 +#endi +#if $data02 != 1 then +# return -1 +#endi +#if $data03 != 13 then +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data05 != 91 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data07 != 13 then +# return -1 +#endi +#if $data08 != @{slop:1.000000, intercept:0.000000}@ then +# return -1 +#endi +#if $data09 != 12 then +# return -1 +#endi # $data0-10 != 3.741657387 # $data0-11 != 1 # $data1-11 != 14 print ====> select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) -sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) -if $rows != 2 then - return -1 -endi -if $data01 != 13 then - return -1 -endi -if $data02 != 1 then - return -1 -endi -if $data03 != 13 then - return -1 -endi -if $data04 != 7 then - return -1 -endi -if $data05 != 91 then - return -1 -endi -if $data06 != 1 then - return -1 -endi -if $data07 != 13 then - return -1 -endi -if $data08 != @{slop:1.000000, intercept:0.000000}@ then - return -1 -endi +#sql select count(*),first(tagtype),last(tagtype),avg(tagtype),sum(tagtype),min(tagtype),max(tagtype),leastsquares(tagtype, 1, 1) from (select * from dev_001 where ts <'2020-05-20 0:0:0') session(ts,1d) +#if $rows != 2 then +# return -1 +#endi +#if $data01 != 13 then +# return -1 +#endi +#if $data02 != 1 then +# return -1 +#endi +#if $data03 != 13 then +# return -1 +#endi +#if $data04 != 7 then +# return -1 +#endi +#if $data05 != 91 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data07 != 13 then +# return -1 +#endi +#if $data08 != @{slop:1.000000, intercept:0.000000}@ then +# return -1 +#endi sql_error select * from dev_001 session(ts,1w) sql_error select count(*) from st session(ts,1w) From 8bcdf8e200132cf066cd3a8b3253f3eecd33c413 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 2 Apr 2022 18:34:17 +0800 Subject: [PATCH 18/32] [TD-14241]: refactor string functions --- source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 126117f982..99a247b45e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -536,6 +536,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { paraType = pParam->node.resType.type; } pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; + break; //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; //int32_t firstParamIndex = 0; //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index bc508f6e22..c02729a084 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -333,15 +333,15 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t inputLen = 0; int32_t numOfRows = 0; for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } } for (int32_t i = 0; i < inputNum; ++i) { - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || - GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { - return TSDB_CODE_FAILED; - } pInputData[i] = pInput[i].columnData; input[i] = pInputData[i]->pData; if (pInput[i].numOfRows == 1) { @@ -402,20 +402,21 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t inputLen = 0; int32_t numOfRows = 0; - for (int32_t i = 0; i < inputNum; ++i) { + for (int32_t i = 1; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[1])) { + return TSDB_CODE_FAILED; + } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } } for (int32_t i = 0; i < inputNum; ++i) { - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || - GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { - return TSDB_CODE_FAILED; - } pInputData[i] = pInput[i].columnData; if (i == 0) { // calculate required separator space - inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2); + int32_t factor = (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) ? TSDB_NCHAR_SIZE : 1; + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2) * factor; } else if (pInput[i].numOfRows == 1) { inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; } else { @@ -429,7 +430,6 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *output = outputBuf; for (int32_t k = 0; k < numOfRows; ++k) { - char *sep = pInputData[0]->pData; if (colDataIsNull_s(pInputData[0], k)) { colDataAppendNULL(pOutputData, k); continue; @@ -437,7 +437,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int16_t dataLen = 0; for (int32_t i = 1; i < inputNum; ++i) { - if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { + if (colDataIsNull_s(pInputData[i], k)) { continue; } @@ -449,6 +449,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p if (i < inputNum - 1) { //insert the separator + char *sep = pInputData[0]->pData; memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep)); dataLen += varDataLen(sep); } From 745fa09bbb903f042755cefb446f3661cb1b793a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 18:58:28 +0800 Subject: [PATCH 19/32] refact transport --- source/dnode/mgmt/bm/inc/bm.h | 31 --------------- source/dnode/mgmt/bm/inc/bmInt.h | 2 +- source/dnode/mgmt/dm/inc/dm.h | 38 ------------------- source/dnode/mgmt/dm/inc/dmInt.h | 3 +- source/dnode/mgmt/dm/src/dmMsg.c | 1 - source/dnode/mgmt/dm/src/dmWorker.c | 5 --- source/dnode/mgmt/main/inc/dnd.h | 30 +++++++++++++++ source/dnode/mgmt/main/inc/dndInt.h | 10 ----- source/dnode/mgmt/main/src/dndStr.c | 2 +- source/dnode/mgmt/main/src/dndTransport.c | 37 ++++-------------- source/dnode/mgmt/mm/inc/mm.h | 35 ----------------- source/dnode/mgmt/mm/inc/mmInt.h | 2 +- source/dnode/mgmt/mm/src/mmInt.c | 8 ---- source/dnode/mgmt/qm/inc/qm.h | 31 --------------- source/dnode/mgmt/qm/inc/qmInt.h | 2 +- source/dnode/mgmt/sm/inc/sm.h | 31 --------------- source/dnode/mgmt/sm/inc/smInt.h | 2 +- source/dnode/mgmt/vm/inc/vm.h | 46 ----------------------- source/dnode/mgmt/vm/inc/vmInt.h | 2 +- 19 files changed, 45 insertions(+), 273 deletions(-) delete mode 100644 source/dnode/mgmt/bm/inc/bm.h delete mode 100644 source/dnode/mgmt/dm/inc/dm.h delete mode 100644 source/dnode/mgmt/mm/inc/mm.h delete mode 100644 source/dnode/mgmt/qm/inc/qm.h delete mode 100644 source/dnode/mgmt/sm/inc/sm.h delete mode 100644 source/dnode/mgmt/vm/inc/vm.h diff --git a/source/dnode/mgmt/bm/inc/bm.h b/source/dnode/mgmt/bm/inc/bm.h deleted file mode 100644 index 79cf76d113..0000000000 --- a/source/dnode/mgmt/bm/inc/bm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_BNODE_H_ -#define _TD_DND_BNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void bmGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/bm/inc/bmInt.h b/source/dnode/mgmt/bm/inc/bmInt.h index f19ba4e034..7a719633fa 100644 --- a/source/dnode/mgmt/bm/inc/bmInt.h +++ b/source/dnode/mgmt/bm/inc/bmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_BNODE_INT_H_ #define _TD_DND_BNODE_INT_H_ -#include "bm.h" +#include "dnd.h" #include "bnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/dm/inc/dm.h b/source/dnode/mgmt/dm/inc/dm.h deleted file mode 100644 index 3984e6dbd4..0000000000 --- a/source/dnode/mgmt/dm/inc/dm.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SDnodeMgmt SDnodeMgmt; - -void dmGetMgmtFp(SMgmtWrapper *pWrapper); -void dmInitMsgHandles(SMgmtWrapper *pWrapper); - -void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index 79d7e926ce..a8f46ef7e2 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_DNODE_INT_H_ #define _TD_DND_DNODE_INT_H_ -#include "dm.h" +#include "dnd.h" #ifdef __cplusplus extern "C" { @@ -45,6 +45,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); // dmMsg.c +void dmInitMsgHandles(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index 02a288e2fd..90117044f9 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" -#include "vm.h" void dmSendStatusReq(SDnodeMgmt *pMgmt) { SDnode *pDnode = pMgmt->pDnode; diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/src/dmWorker.c index 63b9704b78..9045a44232 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/src/dmWorker.c @@ -14,12 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "bm.h" #include "dmInt.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index d948aa305c..b324a279bc 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -165,7 +165,37 @@ int32_t dndInitTrans(SDnode *pDnode); void dndCleanupTrans(SDnode *pDnode); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); +int32_t dndInitMsgHandle(SDnode *pDnode); +// mgmt +void dmGetMgmtFp(SMgmtWrapper *pWrapper); +void bmGetMgmtFp(SMgmtWrapper *pWrapper); +void qmGetMgmtFp(SMgmtWrapper *pMgmt); +void smGetMgmtFp(SMgmtWrapper *pWrapper); +void vmGetMgmtFp(SMgmtWrapper *pWrapper); +void mmGetMgmtFp(SMgmtWrapper *pMgmt); + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); + +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index b4d9de5184..5ef52e3942 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -18,13 +18,6 @@ #include "dnd.h" -#include "bm.h" -#include "dm.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" - #ifdef __cplusplus extern "C" { #endif @@ -47,9 +40,6 @@ void dndHandleEvent(SDnode *pDnode, EDndEvent event); void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); -// dndTransport.c -int32_t dndInitMsgHandle(SDnode *pDnode); - // dndFile.c TdFilePtr dndCheckRunning(const char *dataDir); int32_t dndReadShmFile(SDnode *pDnode); diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/src/dndStr.c index 8a5af68b4f..caaa96de59 100644 --- a/source/dnode/mgmt/main/src/dndStr.c +++ b/source/dnode/mgmt/main/src/dndStr.c @@ -25,7 +25,7 @@ const char *dndStatStr(EDndStatus status) { case DND_STAT_STOPPED: return "stopped"; default: - return "unknown"; + return "UNKNOWN"; } } diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index cefa77890d..2be2da4e46 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -142,20 +142,14 @@ static void dndProcessRequest(SDnode *pDnode, SRpcMsg *pReq, SEpSet *pEpSet) { } } -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); - dndReleaseWrapper(pWrapper); - } - - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); +static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { + SEpSet epSet = {0}; + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); } -static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { +static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { int32_t code = 0; char pass[TSDB_PASSWORD_LEN + 1] = {0}; @@ -183,21 +177,6 @@ static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, ch return 0; } - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { - dndReleaseWrapper(pWrapper); - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - dndReleaseWrapper(pWrapper); - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); @@ -285,11 +264,9 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; - int32_t vgId = pWrapper->msgVgIds[msgIndex]; + int8_t vgId = pWrapper->msgVgIds[msgIndex]; if (msgFp == NULL) continue; - // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); - SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; if (vgId == QND_VGID) { if (pHandle->pQndWrapper != NULL) { diff --git a/source/dnode/mgmt/mm/inc/mm.h b/source/dnode/mgmt/mm/inc/mm.h deleted file mode 100644 index 6ed6c42d93..0000000000 --- a/source/dnode/mgmt/mm/inc/mm.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void mmGetMgmtFp(SMgmtWrapper *pMgmt); - -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); -int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mm/inc/mmInt.h b/source/dnode/mgmt/mm/inc/mmInt.h index 86cba97a33..48c11ae8c4 100644 --- a/source/dnode/mgmt/mm/inc/mmInt.h +++ b/source/dnode/mgmt/mm/inc/mmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_MNODE_INT_H_ #define _TD_DND_MNODE_INT_H_ -#include "mm.h" +#include "dnd.h" #include "mnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/src/mmInt.c index 301ca598e6..bd7446780d 100644 --- a/source/dnode/mgmt/mm/src/mmInt.c +++ b/source/dnode/mgmt/mm/src/mmInt.c @@ -241,14 +241,6 @@ void mmGetMgmtFp(SMgmtWrapper *pWrapper) { pWrapper->fp = mgmtFp; } -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - - int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { SMnodeMgmt *pMgmt = pWrapper->pMgmt; diff --git a/source/dnode/mgmt/qm/inc/qm.h b/source/dnode/mgmt/qm/inc/qm.h deleted file mode 100644 index e28ea3e948..0000000000 --- a/source/dnode/mgmt/qm/inc/qm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void qmGetMgmtFp(SMgmtWrapper *pMgmt); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qm/inc/qmInt.h b/source/dnode/mgmt/qm/inc/qmInt.h index 3e975663d3..b4d57488ef 100644 --- a/source/dnode/mgmt/qm/inc/qmInt.h +++ b/source/dnode/mgmt/qm/inc/qmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_QNODE_INT_H_ #define _TD_DND_QNODE_INT_H_ -#include "qm.h" +#include "dnd.h" #include "qnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/sm/inc/sm.h b/source/dnode/mgmt/sm/inc/sm.h deleted file mode 100644 index 82a52e5d1f..0000000000 --- a/source/dnode/mgmt/sm/inc/sm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_SNODE_H_ -#define _TD_DND_SNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void smGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/sm/inc/smInt.h b/source/dnode/mgmt/sm/inc/smInt.h index 9290384cab..2c9a43cd98 100644 --- a/source/dnode/mgmt/sm/inc/smInt.h +++ b/source/dnode/mgmt/sm/inc/smInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_SNODE_INT_H_ #define _TD_DND_SNODE_INT_H_ -#include "sm.h" +#include "dnd.h" #include "snode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/vm/inc/vm.h b/source/dnode/mgmt/vm/inc/vm.h deleted file mode 100644 index 60d9cfc3a1..0000000000 --- a/source/dnode/mgmt/vm/inc/vm.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -void vmGetMgmtFp(SMgmtWrapper *pWrapper); - -void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); -int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); -void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vm/inc/vmInt.h b/source/dnode/mgmt/vm/inc/vmInt.h index 6722fe1d65..9f8eac9e97 100644 --- a/source/dnode/mgmt/vm/inc/vmInt.h +++ b/source/dnode/mgmt/vm/inc/vmInt.h @@ -17,7 +17,7 @@ #define _TD_DND_VNODES_INT_H_ #include "sync.h" -#include "vm.h" +#include "dnd.h" #include "vnode.h" #ifdef __cplusplus From 6abfd9fed3eb0d0274cde2f790d06a87fb158041 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:21:20 +0800 Subject: [PATCH 20/32] refact transport --- source/dnode/mgmt/main/inc/dndInt.h | 1 - source/dnode/mgmt/main/src/dndEnv.c | 60 +++++++ source/dnode/mgmt/main/src/dndMsg.c | 80 --------- source/dnode/mgmt/main/src/dndStr.c | 77 --------- source/dnode/mgmt/main/src/dndTransport.c | 199 +++++++++++++--------- 5 files changed, 183 insertions(+), 234 deletions(-) delete mode 100644 source/dnode/mgmt/main/src/dndStr.c diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 5ef52e3942..9477bd06b9 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -37,7 +37,6 @@ void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); // dndMsg.c -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndFile.c diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/src/dndEnv.c index 9f75594335..8bdc2867d2 100644 --- a/source/dnode/mgmt/main/src/dndEnv.c +++ b/source/dnode/mgmt/main/src/dndEnv.c @@ -57,3 +57,63 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } + +const char *dndStatStr(EDndStatus status) { + switch (status) { + case DND_STAT_INIT: + return "init"; + case DND_STAT_RUNNING: + return "running"; + case DND_STAT_STOPPED: + return "stopped"; + default: + return "UNKNOWN"; + } +} + +const char *dndNodeLogStr(ENodeType ntype) { + switch (ntype) { + case VNODES: + return "vnode"; + case QNODE: + return "qnode"; + case SNODE: + return "snode"; + case MNODE: + return "mnode"; + case BNODE: + return "bnode"; + default: + return "taosd"; + } +} + +const char *dndNodeProcStr(ENodeType ntype) { + switch (ntype) { + case VNODES: + return "taosv"; + case QNODE: + return "taosq"; + case SNODE: + return "taoss"; + case MNODE: + return "taosm"; + case BNODE: + return "taosb"; + default: + return "taosd"; + } +} + +const char *dndEventStr(EDndEvent ev) { + switch (ev) { + case DND_EVENT_START: + return "start"; + case DND_EVENT_STOP: + return "stop"; + case DND_EVENT_CHILD: + return "child"; + default: + return "UNKNOWN"; + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index 76b66467bf..5d739b7e15 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -16,86 +16,6 @@ #define _DEFAULT_SOURCE #include "dndInt.h" -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); - dndReleaseWrapper(pWrapper); - } -} - -static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - if (msgFp == NULL) { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - - return msgFp; -} - -static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { - SRpcConnInfo connInfo = {0}; - if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); - return -1; - } - - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - pMsg->clientIp = connInfo.clientIp; - pMsg->clientPort = connInfo.clientPort; - memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); - return 0; -} - -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { - int32_t code = -1; - SNodeMsg *pMsg = NULL; - NodeMsgFp msgFp = NULL; - - if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); - } - - if (dndMarkWrapper(pWrapper) != 0) goto _OVER; - if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; - if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; - if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; - - if (pWrapper->procType == PROC_SINGLE) { - dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); - code = (*msgFp)(pWrapper, pMsg); - } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); - } else { - dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - ASSERT(1); - } - -_OVER: - if (code == 0) { - if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is freed in parent process", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - } else { - dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - tmsgSendRsp(&rsp); - } - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - - dndReleaseWrapper(pWrapper); -} - static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/src/dndStr.c deleted file mode 100644 index caaa96de59..0000000000 --- a/source/dnode/mgmt/main/src/dndStr.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -const char *dndStatStr(EDndStatus status) { - switch (status) { - case DND_STAT_INIT: - return "init"; - case DND_STAT_RUNNING: - return "running"; - case DND_STAT_STOPPED: - return "stopped"; - default: - return "UNKNOWN"; - } -} - -const char *dndNodeLogStr(ENodeType ntype) { - switch (ntype) { - case VNODES: - return "vnode"; - case QNODE: - return "qnode"; - case SNODE: - return "snode"; - case MNODE: - return "mnode"; - case BNODE: - return "bnode"; - default: - return "taosd"; - } -} - -const char *dndNodeProcStr(ENodeType ntype) { - switch (ntype) { - case VNODES: - return "taosv"; - case QNODE: - return "taosq"; - case SNODE: - return "taoss"; - case MNODE: - return "taosm"; - case BNODE: - return "taosb"; - default: - return "taosd"; - } -} - -const char *dndEventStr(EDndEvent ev) { - switch (ev) { - case DND_EVENT_START: - return "start"; - case DND_EVENT_STOP: - return "stop"; - case DND_EVENT_CHILD: - return "child"; - default: - return "UNKNOWN"; - } -} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index 2be2da4e46..38bc13fec0 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -20,45 +20,133 @@ #define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" -static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMsgHead *pHead = pMsg->pCont; - int32_t vgId = htonl(pHead->vgId); - - SMgmtWrapper *pWrapper = pHandle->pWrapper; - if (vgId == QND_VGID) { - pWrapper = pHandle->pQndWrapper; - } else if (vgId == MND_VGID) { - pWrapper = pHandle->pMndWrapper; - } - - dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, - pMsg->handle, pMsg->ahandle, vgId); - dndProcessRpcMsg(pWrapper, pMsg, pEpSet); +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); } -static void dndProcessResponse(SDnode *pDnode, SRpcMsg *pRsp, SEpSet *pEpSet) { - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pRsp->msgType; +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + pMsg->clientIp = connInfo.clientIp; + pMsg->clientPort = connInfo.clientPort; + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + return 0; +} + +static void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + if (pWrapper->procType == PROC_SINGLE) { + dTrace("msg:%p, is created, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = (*msgFp)(pWrapper, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is created and put into child queue, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); + } else { + dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + ASSERT(1); + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed in parent process", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + tmsgSendRsp(&rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static void dndProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pMsg->msgType; + bool isReq = msgType & 1u; + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + SMgmtWrapper *pWrapper = pHandle->pWrapper; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dndProcessStartupReq(pDnode, pMsg); return; } - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), - pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); - dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pRsp, pEpSet); + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); } - } else { - dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); + rpcFreeCont(pMsg->pCont); + return; } + + if (isReq && pMsg->pCont == NULL) { + dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + if (pWrapper == NULL) { + dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + } + rpcFreeCont(pMsg->pCont); + } + + if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { + SMsgHead *pHead = pMsg->pCont; + int32_t vgId = ntohl(pHead->vgId); + if (vgId == QND_VGID) { + pWrapper = pHandle->pQndWrapper; + } else if (vgId == MND_VGID) { + pWrapper = pHandle->pMndWrapper; + } else { + } + } + + dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle); + dndProcessRpcMsg(pWrapper, pMsg, pEpSet); } static int32_t dndInitClient(SDnode *pDnode) { @@ -68,7 +156,7 @@ static int32_t dndInitClient(SDnode *pDnode) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND"; rpcInit.numOfThreads = 1; - rpcInit.cfp = (RpcCfp)dndProcessResponse; + rpcInit.cfp = (RpcCfp)dndProcessMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; @@ -100,48 +188,6 @@ static void dndCleanupClient(SDnode *pDnode) { } } -static void dndProcessRequest(SDnode *pDnode, SRpcMsg *pReq, SEpSet *pEpSet) { - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pReq->msgType; - - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, - pReq->handle, pReq->ahandle); - dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pReq, pEpSet); - } - } else { - dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { SEpSet epSet = {0}; SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; @@ -149,7 +195,8 @@ static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); } -static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { +static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, + char *ckey) { int32_t code = 0; char pass[TSDB_PASSWORD_LEN + 1] = {0}; @@ -219,7 +266,7 @@ static int32_t dndInitServer(SDnode *pDnode) { rpcInit.localPort = pDnode->serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = (RpcCfp)dndProcessRequest; + rpcInit.cfp = (RpcCfp)dndProcessMsg; rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; From b2bf8539527bf4c200faf63ca3bcfdf08f091d7b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:22:59 +0800 Subject: [PATCH 21/32] rename file --- source/dnode/mgmt/dm/src/dmMsg.c | 101 ++++++++++++++++++++++++ source/dnode/mgmt/main/src/dndMsg.c | 117 ---------------------------- 2 files changed, 101 insertions(+), 117 deletions(-) delete mode 100644 source/dnode/mgmt/main/src/dndMsg.c diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index 90117044f9..979db7aac0 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -117,6 +117,107 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} + void dmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID); diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c deleted file mode 100644 index 5d739b7e15..0000000000 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper != NULL) { - dndReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; - dError("failed to create node since %s", terrstr()); - return -1; - } - - pWrapper = &pDnode->wrappers[ntype]; - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); - return -1; - } - - int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); - if (code != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); - } else { - dDebug("node:%s, has been opened", pWrapper->name); - pWrapper->deployed = true; - } - - return code; -} - -static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - dError("failed to drop node since %s", terrstr()); - return -1; - } - - taosWLockLatch(&pWrapper->latch); - pWrapper->deployed = false; - - int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); - if (code != 0) { - pWrapper->deployed = true; - dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); - } else { - pWrapper->deployed = false; - dDebug("node:%s, has been dropped", pWrapper->name); - } - - taosWUnLockLatch(&pWrapper->latch); - dndReleaseWrapper(pWrapper); - return code; -} - -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { - switch (pMsg->rpcMsg.msgType) { - case TDMT_DND_CREATE_MNODE: - return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_DROP_MNODE: - return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_CREATE_QNODE: - return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_DROP_QNODE: - return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_CREATE_SNODE: - return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_DROP_SNODE: - return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_CREATE_BNODE: - return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); - case TDMT_DND_DROP_BNODE: - return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - return -1; - } -} - -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); -} From a554bd7fe7ceb370a3d9e4deaf6d243434eeb94b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:24:20 +0800 Subject: [PATCH 22/32] rename file --- source/dnode/mgmt/bm/src/{bmMsg.c => bmHandle.c} | 0 source/dnode/mgmt/dm/src/{dmMsg.c => dmHandle.c} | 0 source/dnode/mgmt/mm/src/{mmMsg.c => mmHandle.c} | 0 source/dnode/mgmt/qm/src/{qmMsg.c => qmHandle.c} | 0 source/dnode/mgmt/sm/src/{smMsg.c => smHandle.c} | 0 source/dnode/mgmt/vm/src/{vmMsg.c => vmHandle.c} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename source/dnode/mgmt/bm/src/{bmMsg.c => bmHandle.c} (100%) rename source/dnode/mgmt/dm/src/{dmMsg.c => dmHandle.c} (100%) rename source/dnode/mgmt/mm/src/{mmMsg.c => mmHandle.c} (100%) rename source/dnode/mgmt/qm/src/{qmMsg.c => qmHandle.c} (100%) rename source/dnode/mgmt/sm/src/{smMsg.c => smHandle.c} (100%) rename source/dnode/mgmt/vm/src/{vmMsg.c => vmHandle.c} (100%) diff --git a/source/dnode/mgmt/bm/src/bmMsg.c b/source/dnode/mgmt/bm/src/bmHandle.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmMsg.c rename to source/dnode/mgmt/bm/src/bmHandle.c diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmHandle.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmMsg.c rename to source/dnode/mgmt/dm/src/dmHandle.c diff --git a/source/dnode/mgmt/mm/src/mmMsg.c b/source/dnode/mgmt/mm/src/mmHandle.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmMsg.c rename to source/dnode/mgmt/mm/src/mmHandle.c diff --git a/source/dnode/mgmt/qm/src/qmMsg.c b/source/dnode/mgmt/qm/src/qmHandle.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmMsg.c rename to source/dnode/mgmt/qm/src/qmHandle.c diff --git a/source/dnode/mgmt/sm/src/smMsg.c b/source/dnode/mgmt/sm/src/smHandle.c similarity index 100% rename from source/dnode/mgmt/sm/src/smMsg.c rename to source/dnode/mgmt/sm/src/smHandle.c diff --git a/source/dnode/mgmt/vm/src/vmMsg.c b/source/dnode/mgmt/vm/src/vmHandle.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmMsg.c rename to source/dnode/mgmt/vm/src/vmHandle.c From 7e361ebcd09a9064fed94b65a3bfdff34f30bf8f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:52:04 +0800 Subject: [PATCH 23/32] refact dm --- include/util/tdef.h | 6 +- source/dnode/mgmt/dm/inc/dmInt.h | 7 +- source/dnode/mgmt/dm/src/dmHandle.c | 37 ++--- source/dnode/mgmt/dm/src/dmInt.c | 2 +- source/dnode/mgmt/dm/src/dmWorker.c | 53 +++---- source/dnode/mgmt/main/src/dndInt.c | 7 + source/dnode/mgmt/main/src/dndTransport.c | 8 +- source/dnode/mgmt/mm/src/mmHandle.c | 162 ++++++++++----------- source/dnode/mgmt/qm/src/qmHandle.c | 18 +-- source/dnode/mgmt/sm/src/smHandle.c | 4 +- source/dnode/mgmt/vm/src/vmHandle.c | 88 +++++------ source/dnode/mnode/impl/src/mndQuery.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 2 +- 13 files changed, 200 insertions(+), 196 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index be88afcf66..f1af9eca1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -480,9 +480,9 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define MND_VGID -1 -#define QND_VGID 1 -#define VND_VGID 0 +#define MNODE_HANDLE -1 +#define QNODE_HANDLE 1 +#define DEFAULT_HANDLE 0 #define MAX_NUM_STR_SIZE 40 diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index a8f46ef7e2..98abb8ef54 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -45,7 +45,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); // dmMsg.c -void dmInitMsgHandles(SMgmtWrapper *pWrapper); +void dmInitMsgHandle(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); @@ -53,10 +53,11 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); // dmWorker.c +int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt); -int32_t dmStartThread(SDnodeMgmt *pMgmt); -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmPutMsgToMgmtWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmPutMsgToStatusWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/dm/src/dmHandle.c b/source/dnode/mgmt/dm/src/dmHandle.c index 979db7aac0..07eca3f230 100644 --- a/source/dnode/mgmt/dm/src/dmHandle.c +++ b/source/dnode/mgmt/dm/src/dmHandle.c @@ -195,14 +195,7 @@ int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { } } -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { +static void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); } @@ -218,21 +211,21 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } -void dmInitMsgHandles(SMgmtWrapper *pWrapper) { +void dmInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmPutMsgToStatusWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index 8c46c402ab..1e38b8304d 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -165,7 +165,7 @@ void dmGetMgmtFp(SMgmtWrapper *pWrapper) { mgmtFp.startFp = dmStart; mgmtFp.requiredFp = dmRequire; - dmInitMsgHandles(pWrapper); + dmInitMsgHandle(pWrapper); pWrapper->name = "dnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/src/dmWorker.c index 9045a44232..6061189ca1 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/src/dmWorker.c @@ -18,7 +18,7 @@ static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; int64_t lastStatusTime = taosGetTimestampMs(); int64_t lastMonitorTime = lastStatusTime; @@ -32,8 +32,7 @@ static void *dmThreadRoutine(void *param) { } int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; + float statusInterval = (curTime - lastStatusTime) / 1000.0f; if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { dmSendStatusReq(pMgmt); lastStatusTime = curTime; @@ -47,10 +46,21 @@ static void *dmThreadRoutine(void *param) { } } +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; dTrace("msg:%p, will be processed in dnode queue", pMsg); @@ -95,16 +105,14 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { } int32_t dmStartWorker(SDnodeMgmt *pMgmt) { - SSingleWorkerCfg mgmtCfg = { - .min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) { + SSingleWorkerCfg mcfg = {.min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mcfg) != 0) { dError("failed to start dnode mgmt worker since %s", terrstr()); return -1; } - SSingleWorkerCfg statusCfg = { - .min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) { + SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) { dError("failed to start dnode status worker since %s", terrstr()); return -1; } @@ -113,17 +121,6 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return 0; } -int32_t dmStartThread(SDnodeMgmt *pMgmt) { - pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - void dmStopWorker(SDnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->statusWorker); @@ -135,12 +132,18 @@ void dmStopWorker(SDnodeMgmt *pMgmt) { dDebug("dnode workers are closed"); } -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { +int32_t dmPutMsgToMgmtWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } + + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + +int32_t dmPutMsgToStatusWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->statusWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 5e4b0a1f7b..72feed7c62 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -204,3 +204,10 @@ void dndSetStatus(SDnode *pDnode, EDndStatus status) { pDnode->status = status; } } + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index 38bc13fec0..babc6d1209 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -137,9 +137,9 @@ static void dndProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { SMsgHead *pHead = pMsg->pCont; int32_t vgId = ntohl(pHead->vgId); - if (vgId == QND_VGID) { + if (vgId == QNODE_HANDLE) { pWrapper = pHandle->pQndWrapper; - } else if (vgId == MND_VGID) { + } else if (vgId == MNODE_HANDLE) { pWrapper = pHandle->pMndWrapper; } else { } @@ -315,13 +315,13 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { if (msgFp == NULL) continue; SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; - if (vgId == QND_VGID) { + if (vgId == QNODE_HANDLE) { if (pHandle->pQndWrapper != NULL) { dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); return -1; } pHandle->pQndWrapper = pWrapper; - } else if (vgId == MND_VGID) { + } else if (vgId == MNODE_HANDLE) { if (pHandle->pMndWrapper != NULL) { dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); return -1; diff --git a/source/dnode/mgmt/mm/src/mmHandle.c b/source/dnode/mgmt/mm/src/mmHandle.c index 6afcd249b3..666f9daa91 100644 --- a/source/dnode/mgmt/mm/src/mmHandle.c +++ b/source/dnode/mgmt/mm/src/mmHandle.c @@ -75,91 +75,91 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); } diff --git a/source/dnode/mgmt/qm/src/qmHandle.c b/source/dnode/mgmt/qm/src/qmHandle.c index da5ba6472a..14ac08af05 100644 --- a/source/dnode/mgmt/qm/src/qmHandle.c +++ b/source/dnode/mgmt/qm/src/qmHandle.c @@ -56,14 +56,14 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void qmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QNODE_HANDLE); } diff --git a/source/dnode/mgmt/sm/src/smHandle.c b/source/dnode/mgmt/sm/src/smHandle.c index c522ef7fc3..dc5ee1155a 100644 --- a/source/dnode/mgmt/sm/src/smHandle.c +++ b/source/dnode/mgmt/sm/src/smHandle.c @@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void smInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by SNODE - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/vm/src/vmHandle.c b/source/dnode/mgmt/vm/src/vmHandle.c index c3ad74d246..6a51c9a77f 100644 --- a/source/dnode/mgmt/vm/src/vmHandle.c +++ b/source/dnode/mgmt/vm/src/vmHandle.c @@ -235,49 +235,49 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { void vmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 40b4f60bd4..d184e354c4 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -52,7 +52,7 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) { } int32_t mndInitQuery(SMnode *pMnode) { - if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { + if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { mError("failed to init qworker in mnode since %s", terrstr()); return -1; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 11baeff04a..5e466c8cda 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -419,7 +419,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - SQueryNodeAddr addr = { .nodeId = MND_VGID, .epSet = pCxt->pPlanCxt->mgmtEpSet }; + SQueryNodeAddr addr = { .nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet }; taosArrayPush(pCxt->pExecNodeList, &addr); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; From 469cd23f43c38f3dbf9112eb557eceb4181380dd Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Apr 2022 07:57:44 -0400 Subject: [PATCH 24/32] integrate constant calculate --- include/libs/nodes/nodes.h | 16 +-- include/libs/nodes/querynodes.h | 1 + source/libs/executor/src/executorimpl.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 25 +--- source/libs/nodes/src/nodesCodeFuncs.c | 6 + source/libs/nodes/src/nodesTraverseFuncs.c | 48 ++++---- source/libs/nodes/test/nodesTest.cpp | 2 +- source/libs/parser/CMakeLists.txt | 2 +- source/libs/parser/inc/parInt.h | 5 +- source/libs/parser/src/parAstCreater.c | 36 +++--- source/libs/parser/src/parAstParser.c | 2 +- source/libs/parser/src/parCalcConst.c | 128 +++++++++++++++++++++ source/libs/parser/src/parTranslater.c | 31 ++--- source/libs/parser/src/parser.c | 7 +- source/libs/parser/test/parserAstTest.cpp | 4 +- source/libs/planner/src/planLogicCreater.c | 4 +- source/libs/planner/src/planPhysiCreater.c | 6 +- source/libs/planner/test/plannerTest.cpp | 7 ++ source/libs/scalar/src/filter.c | 8 +- source/libs/scalar/src/scalar.c | 4 +- 20 files changed, 234 insertions(+), 110 deletions(-) create mode 100644 source/libs/parser/src/parCalcConst.c diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9668082696..d2217f7e01 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -213,16 +213,16 @@ typedef enum EDealRes { } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNodeptr a, const SNodeptr b); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index ca4646a5ca..a8ff48072f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -79,6 +79,7 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; + bool genByCalc; union { bool b; int64_t i; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 169e47eb33..a0555ab8cd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5801,7 +5801,7 @@ void getDBNameFromCondition(SNode* pCondition, char* dbName) { return; } - nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName); + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); } static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f8bc99b975..ac78988900 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -113,36 +113,13 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); - COPY_SCALAR_FIELD(isDuration); - COPY_SCALAR_FIELD(translate); if (!pSrc->translate) { return (SNode*)pDst; } switch (pSrc->node.resType.type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - COPY_SCALAR_FIELD(datum.b); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - COPY_SCALAR_FIELD(datum.i); - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - COPY_SCALAR_FIELD(datum.u); - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - COPY_SCALAR_FIELD(datum.d); - break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 5337965164..e39a7ecfa6 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1544,6 +1544,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkValueTranslate, pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -1614,6 +1617,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index bbd0473edd..15c078d8d2 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -138,19 +138,19 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke return DEAL_RES_CONTINUE; } -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { +void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); } -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } @@ -267,19 +267,19 @@ static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRe return DEAL_RES_CONTINUE; } -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext); } -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext); } @@ -290,20 +290,20 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa switch (clause) { case SQL_CLAUSE_FROM: - nodesWalkNode(pSelect->pFromTable, walker, pContext); - nodesWalkNode(pSelect->pWhere, walker, pContext); + nodesWalkExpr(pSelect->pFromTable, walker, pContext); + nodesWalkExpr(pSelect->pWhere, walker, pContext); case SQL_CLAUSE_WHERE: - nodesWalkList(pSelect->pPartitionByList, walker, pContext); + nodesWalkExprs(pSelect->pPartitionByList, walker, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesWalkNode(pSelect->pWindow, walker, pContext); + nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: - nodesWalkList(pSelect->pGroupByList, walker, pContext); + nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: - nodesWalkNode(pSelect->pHaving, walker, pContext); + nodesWalkExpr(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: - nodesWalkList(pSelect->pOrderByList, walker, pContext); + nodesWalkExprs(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: - nodesWalkList(pSelect->pProjectionList, walker, pContext); + nodesWalkExprs(pSelect->pProjectionList, walker, pContext); case SQL_CLAUSE_SELECT: default: break; @@ -319,20 +319,20 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit switch (clause) { case SQL_CLAUSE_FROM: - nodesRewriteNode(&(pSelect->pFromTable), rewriter, pContext); - nodesRewriteNode(&(pSelect->pWhere), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pFromTable), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext); case SQL_CLAUSE_WHERE: - nodesRewriteList(pSelect->pPartitionByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesRewriteNode(&(pSelect->pWindow), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: - nodesRewriteList(pSelect->pGroupByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: - nodesRewriteNode(&(pSelect->pHaving), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); case SQL_CLAUSE_HAVING: - nodesRewriteList(pSelect->pProjectionList, rewriter, pContext); + nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext); case SQL_CLAUSE_SELECT: - nodesRewriteList(pSelect->pOrderByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); default: break; } diff --git a/source/libs/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp index fd4a9e8c20..fdfb9522db 100644 --- a/source/libs/nodes/test/nodesTest.cpp +++ b/source/libs/nodes/test/nodesTest.cpp @@ -49,7 +49,7 @@ TEST(NodesTest, traverseTest) { EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); EDealRes res = DEAL_RES_CONTINUE; - nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res); + nodesRewriteExprPostOrder(&pRoot, rewriterTest, &res); EXPECT_EQ(res, DEAL_RES_CONTINUE); EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE); EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18"); diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 5d02868657..c3157480a9 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function transport qcom + PRIVATE os util nodes catalog function scalar transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 9bad3e9eb9..ea01caf9d7 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -23,9 +23,10 @@ extern "C" { #include "parser.h" int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery); -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 18a0ae8cfa..07316ddd39 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -243,27 +243,27 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio return pOptions; } - char val[20] = {0}; - int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val)); - char* pStart = val; - char* pEnd = val + len; - int32_t sepOrder = 1; - while (1) { - char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':'); - SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { - pCxt->valid = false; - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); - return pOptions; + if (pVal->n > 2) { + char* pStart = pVal->z + 1; + char* pEnd = pVal->z + pVal->n - 1; + int32_t sepOrder = 1; + while (1) { + char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':'); + SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { + pCxt->valid = false; + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); + return pOptions; + } + if (NULL == pPos) { + break; + } + pStart = pPos + 1; } - if (NULL == pPos) { - break; - } - pStart = pPos + 1; } - if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val); + if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z); pCxt->valid = false; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5c38ccaff8..76af6b7ac7 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -26,7 +26,7 @@ extern void Parse(void*, int, SToken, void*); extern void ParseFree(void*, FFree); extern void ParseTrace(FILE*, char*); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c new file mode 100644 index 0000000000..a572ccf5bd --- /dev/null +++ b/source/libs/parser/src/parCalcConst.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parInt.h" +#include "scalar.h" + +typedef struct SCalcConstContext { + int32_t code; +} SCalcConstContext; + +static int32_t calcConstQuery(SNode* pStmt); + +static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { + SNode* pNew = NULL; + pCxt->code = scalarCalculateConstants(*pNode, &pNew); + if (TSDB_CODE_SUCCESS != pCxt->code) { + return DEAL_RES_ERROR; + } + ((SValueNode*)pNew)->genByCalc = true; + ((SValueNode*)pNew)->translate = true; + *pNode = pNew; + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { + SOperatorNode* pOp = *pNode; + if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); + } + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { + SFunctionNode* pFunc = *pNode; + SNode* pParam = NULL; + FOREACH(pParam, pFunc->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) { + SLogicConditionNode* pCond = *pNode; + SNode* pParam = NULL; + FOREACH(pParam, pCond->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) { + SCalcConstContext* pCxt = pContext; + pCxt->code = calcConstQuery((*pNode)->pSubquery); + return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + +static EDealRes calcConst(SNode** pNode, void* pContext) { + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + return calcConstOperator((SOperatorNode**)pNode, pContext); + case QUERY_NODE_FUNCTION: + return calcConstFunction((SFunctionNode**)pNode, pContext); + case QUERY_NODE_LOGIC_CONDITION: + return calcConstLogicCond((SLogicConditionNode**)pNode, pContext); + case QUERY_NODE_TEMP_TABLE: + return calcConstSubquery((STempTableNode**)pNode, pContext); + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t calcConstSelect(SSelectStmt* pSelect) { + SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS }; + nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); + } + return cxt.code; +} + +static int32_t calcConstQuery(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return calcConstSelect((SSelectStmt*)pStmt); + default: + break; + } + return TSDB_CODE_SUCCESS; +} + +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { + return calcConstQuery(pQuery->pRoot); +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 641750cd98..850ef9a4ac 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -50,14 +50,12 @@ static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { - va_list vArgList; - va_start(vArgList, errCode); - generateSyntaxErrMsg(&pCxt->msgBuf, errCode, vArgList); - va_end(vArgList); - pCxt->errCode = errCode; - return DEAL_RES_ERROR; -} +#define generateDealNodeErrMsg(pCxt, code, ...) \ + ({ \ + generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ + pCxt->errCode = code; \ + DEAL_RES_ERROR; \ + }) static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); @@ -440,6 +438,9 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (nodesIsUnaryOp(pOp)) { + if (OP_TYPE_MINUS == pOp->opType && !IS_NUMERIC_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + } return DEAL_RES_CONTINUE; } SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; @@ -510,12 +511,12 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { } static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + nodesWalkExprPostOrder(pNode, doTranslateExpr, pCxt); return pCxt->errCode; } static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { - nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + nodesWalkExprsPostOrder(pList, doTranslateExpr, pCxt); return pCxt->errCode; } @@ -570,7 +571,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); + nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -578,7 +579,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkList(pList, doCheckExprForGroupBy, pCxt); + nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -605,9 +606,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; - nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (cxt.existAggFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); @@ -2633,7 +2634,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = { .pParseCxt = pParseCxt, .errCode = TSDB_CODE_SUCCESS, diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d410aa2e17..ebe76cc129 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -30,9 +30,12 @@ static bool isInsertSql(const char* pStr, size_t length) { } static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { - int32_t code = doParse(pCxt, pQuery); + int32_t code = parse(pCxt, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = doTranslate(pCxt, *pQuery); + code = translate(pCxt, *pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = calculateConstant(pCxt, *pQuery); } return code; } diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 6dc3b53ae3..72670bb79a 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -54,7 +54,7 @@ private: static const int max_err_len = 1024; bool runImpl(int32_t parseCode, int32_t translateCode) { - int32_t code = doParse(&cxt_, &query_); + int32_t code = parse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == parseCode); @@ -63,7 +63,7 @@ private: return false; } parsedAstStr_ = toString(query_->pRoot); - code = doTranslate(&cxt_, query_); + code = translate(&cxt_, query_); if (code != TSDB_CODE_SUCCESS) { translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == translateCode); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f57f476d51..149071bad2 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -83,7 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - nodesWalkList(pExprs, doNameExpr, NULL); + nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -384,7 +384,7 @@ static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pEx return TSDB_CODE_OUT_OF_MEMORY; } - nodesWalkList(pExprs, doCreateColumn, &cxt); + nodesWalkExprs(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); return cxt.errCode; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 11baeff04a..4e729a051f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -264,7 +264,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkNode(pRes, doSetSlotId, &cxt); + nodesWalkExpr(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return cxt.errCode; @@ -285,7 +285,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkList(pRes, doSetSlotId, &cxt); + nodesWalkExprs(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return cxt.errCode; @@ -606,7 +606,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } } SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; - nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { nodesDestroyList(cxt.pPrecalcExprs); *pPrecalcExprs = NULL; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index ad76c8f879..32b221c4fb 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -157,6 +157,13 @@ TEST_F(PlannerTest, simple) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, selectConstant) { + setDatabase("root", "test"); + + bind("SELECT 2-1 FROM t1"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, stSimple) { setDatabase("root", "test"); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b0632bbc34..23a22dfabf 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1297,7 +1297,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { resGroup = taosArrayInit(4, sizeof(SFilterGroup)); SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); @@ -1322,7 +1322,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { if (LOGIC_COND_TYPE_OR == node->condType) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext); FLT_ERR_JRET(ctx->code); cell = cell->pNext; @@ -3190,7 +3190,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { filterInitUnitsFields(info); SFltBuildGroupCtx tctx = {.info = info, .group = group}; - nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); @@ -3566,7 +3566,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat); FLT_RET(pStat->code); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 1cc259d4da..08ad7bfa29 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -673,7 +673,7 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); + nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); SCL_ERR_JRET(ctx.code); *pRes = pNode; @@ -696,7 +696,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); + nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); SCL_ERR_JRET(ctx.code); if (pDst) { From 8e2d121b3ad4ce4e5f73907f1543fef1e0557586 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 19:59:25 +0800 Subject: [PATCH 25/32] temp --- source/client/src/tmq.c | 55 ++++++++++++++-------- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- source/util/src/tlog.c | 8 ++-- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 426a62433b..81219e1c36 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -79,6 +79,7 @@ struct tmq_t { tmq_commit_cb* commit_cb; int32_t nextTopicIdx; int8_t epStatus; + int32_t epSkipCnt; int32_t waitingRequest; int32_t readyRequest; SArray* clientTopics; // SArray @@ -107,6 +108,7 @@ typedef struct { // connection info int32_t vgId; int32_t vgStatus; + int64_t skipCnt; SEpSet epSet; } SMqClientVg; @@ -313,6 +315,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->waitingRequest = 0; pTmq->readyRequest = 0; pTmq->epStatus = 0; + pTmq->epSkipCnt = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -348,6 +351,8 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->epoch = 0; pTmq->waitingRequest = 0; pTmq->readyRequest = 0; + pTmq->epStatus = 0; + pTmq->epSkipCnt = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -842,7 +847,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); return 0; } @@ -892,26 +897,26 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset, + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset, pRsp->msg.rspOffset); pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return 0; CREATE_MSG_FAIL: if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return code; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ - tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch); + tscDebug("consumer %ld update ep epoch %d to epoch %d", tmq->consumerId, tmq->epoch, epoch); bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; @@ -942,7 +947,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d build %s\n", tmq->consumerId, epoch, pVgCur->vgId, vgKey); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -956,18 +961,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset og to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d found %s\n", tmq->consumerId, epoch, pVgEp->vgId, vgKey); } - tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, + .skipCnt = 0, }; taosArrayPush(topic.vgs, &clientVg); set = true; @@ -984,9 +990,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; - tscDebug("consumer %ld recv ep", tmq->consumerId); if (code != 0) { - tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync); + tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); goto END; } @@ -995,6 +1000,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // Epoch will only increase when received newer epoch ep msg SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); + tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); if (head->epoch <= epoch) { goto END; } @@ -1019,7 +1025,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp); taosWriteQitem(tmq->mqueue, pRsp); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ } END: @@ -1033,9 +1039,11 @@ END: int32_t tmqAskEp(tmq_t* tmq, bool sync) { int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); if (epStatus == 1) { - tscDebug("consumer %ld skip ask ep", tmq->consumerId); - return 0; + int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); + tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); + if (epSkipCnt < 40) return 0; } + atomic_store_32(&tmq->epSkipCnt, 0); int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { @@ -1221,20 +1229,29 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { - tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId); + int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1); + tscDebug("consumer %ld skip vg %d skip cnt %ld", tmq->consumerId, pVg->vgId, skipCnt); continue; +#if 0 + if (skipCnt < 30000) { + continue; + } else { + tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId); + } +#endif } + atomic_store_64(&pVg->skipCnt, 0); SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } pParam->tmq = tmq; @@ -1247,7 +1264,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { taosMemoryFree(pReq); taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } @@ -1379,7 +1396,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmqAskEp(tmq, false); tmqPollImpl(tmq, blocking_time); - tsem_wait(&tmq->rspSem); + /*tsem_wait(&tmq->rspSem);*/ rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); if (rspMsg) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 211163ce35..ae9a4198a4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -216,7 +216,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { // TODO int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mTrace("try to get sub ep, old val: %d", hbStatus); + mTrace("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, pConsumer->epoch, hbStatus); atomic_store_32(&pConsumer->hbStatus, 0); /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index ef15f44f8f..f6fec47196 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -214,6 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; + taosSsleep(2); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); @@ -347,12 +348,13 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); + TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->pFile == NULL) { + if (pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } + tsLogObj.logHandle->pFile = pFile; taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines @@ -746,4 +748,4 @@ void taosSetAllDebugFlag(int32_t flag) { fsDebugFlag = flag; uInfo("all debug flag are set to %d", flag); -} \ No newline at end of file +} From e00f74ccbc3f7f1b33e91c5a0ace429120d7b1b4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 20:05:42 +0800 Subject: [PATCH 26/32] refact dm --- source/dnode/mgmt/bm/src/bmInt.c | 2 +- source/dnode/mgmt/dm/inc/dmInt.h | 9 +++- source/dnode/mgmt/dm/src/dmHandle.c | 22 +++++----- source/dnode/mgmt/dm/src/dmInt.c | 8 ++-- source/dnode/mgmt/dm/src/dmWorker.c | 14 +----- source/dnode/mgmt/main/inc/dnd.h | 67 ++++++++++++++++++++--------- source/dnode/mgmt/main/src/dndInt.c | 12 +++--- source/dnode/mgmt/mm/src/mmInt.c | 2 +- source/dnode/mgmt/qm/src/qmInt.c | 2 +- source/dnode/mgmt/sm/src/smInt.c | 2 +- source/dnode/mgmt/vm/src/vmInt.c | 2 +- 11 files changed, 83 insertions(+), 59 deletions(-) diff --git a/source/dnode/mgmt/bm/src/bmInt.c b/source/dnode/mgmt/bm/src/bmInt.c index 4ca7afbb6d..2cb0f50dfe 100644 --- a/source/dnode/mgmt/bm/src/bmInt.c +++ b/source/dnode/mgmt/bm/src/bmInt.c @@ -109,7 +109,7 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) { return code; } -void bmGetMgmtFp(SMgmtWrapper *pWrapper) { +void bmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = bmOpen; mgmtFp.closeFp = bmClose; diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index 98abb8ef54..e68f7ddb65 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -39,18 +39,25 @@ typedef struct SDnodeMgmt { SMgmtWrapper *pWrapper; } SDnodeMgmt; +// dmInt.c +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); + // dmFile.c int32_t dmReadFile(SDnodeMgmt *pMgmt); int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); -// dmMsg.c +// dmHandle.c void dmInitMsgHandle(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessCDnodeMsg(SDnode *pDnode, SNodeMsg *pMsg); // dmWorker.c int32_t dmStartThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/dm/src/dmHandle.c b/source/dnode/mgmt/dm/src/dmHandle.c index 07eca3f230..6492995c5c 100644 --- a/source/dnode/mgmt/dm/src/dmHandle.c +++ b/source/dnode/mgmt/dm/src/dmHandle.c @@ -118,7 +118,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { } -static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { +static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { dndReleaseWrapper(pWrapper); @@ -146,7 +146,7 @@ static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg return code; } -static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { +static int32_t dmProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper == NULL) { terrno = TSDB_CODE_NODE_NOT_DEPLOYED; @@ -171,24 +171,24 @@ static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg * return code; } -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { +int32_t dmProcessCDnodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { switch (pMsg->rpcMsg.msgType) { case TDMT_DND_CREATE_MNODE: - return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, MNODE, pMsg); case TDMT_DND_DROP_MNODE: - return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, MNODE, pMsg); case TDMT_DND_CREATE_QNODE: - return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, QNODE, pMsg); case TDMT_DND_DROP_QNODE: - return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, QNODE, pMsg); case TDMT_DND_CREATE_SNODE: - return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, SNODE, pMsg); case TDMT_DND_DROP_SNODE: - return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, SNODE, pMsg); case TDMT_DND_CREATE_BNODE: - return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, BNODE, pMsg); case TDMT_DND_DROP_BNODE: - return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, BNODE, pMsg); default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; return -1; diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index 1e38b8304d..c710af9006 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -78,7 +78,7 @@ static int32_t dmStart(SMgmtWrapper *pWrapper) { return dmStartThread(pWrapper->pMgmt); } -int32_t dmInit(SMgmtWrapper *pWrapper) { +static int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); dInfo("dnode-mgmt start to init"); @@ -124,7 +124,7 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return 0; } -void dmCleanup(SMgmtWrapper *pWrapper) { +static void dmCleanup(SMgmtWrapper *pWrapper) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; if (pMgmt == NULL) return; @@ -153,12 +153,12 @@ void dmCleanup(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt is cleaned up"); } -int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { +static int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { *required = true; return 0; } -void dmGetMgmtFp(SMgmtWrapper *pWrapper) { +void dmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = dmInit; mgmtFp.closeFp = dmCleanup; diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/src/dmWorker.c index 6061189ca1..40ca858183 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/src/dmWorker.c @@ -66,16 +66,6 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { dTrace("msg:%p, will be processed in dnode queue", pMsg); switch (pRpc->msgType) { - case TDMT_DND_CREATE_MNODE: - case TDMT_DND_CREATE_QNODE: - case TDMT_DND_CREATE_SNODE: - case TDMT_DND_CREATE_BNODE: - case TDMT_DND_DROP_MNODE: - case TDMT_DND_DROP_QNODE: - case TDMT_DND_DROP_SNODE: - case TDMT_DND_DROP_BNODE: - code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); - break; case TDMT_DND_CONFIG_DNODE: code = dmProcessConfigReq(pMgmt, pMsg); break; @@ -89,8 +79,8 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { code = dmProcessGrantRsp(pMgmt, pMsg); break; default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + code = dmProcessCDnodeMsg(pMgmt->pDnode, pMsg); + break; } if (pRpc->msgType & 1u) { diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index b324a279bc..449c4399db 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -27,13 +27,13 @@ #include "tlockfree.h" #include "tlog.h" #include "tmsg.h" +#include "tmsgcb.h" #include "tprocess.h" #include "tqueue.h" #include "trpc.h" #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tmsgcb.h" #include "dnode.h" #include "monitor.h" @@ -42,12 +42,42 @@ extern "C" { #endif -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} +#define dFatal(...) \ + { \ + if (dDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ + } +#define dError(...) \ + { \ + if (dDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ + } +#define dWarn(...) \ + { \ + if (dDebugFlag & DEBUG_WARN) { \ + taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ + } +#define dInfo(...) \ + { \ + if (dDebugFlag & DEBUG_INFO) { \ + taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ + } +#define dDebug(...) \ + { \ + if (dDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); \ + } \ + } +#define dTrace(...) \ + { \ + if (dDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); \ + } \ + } typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; @@ -73,7 +103,7 @@ typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); typedef struct SMsgHandle { - SMgmtWrapper *pQndWrapper; + SMgmtWrapper *pQndWrapper; SMgmtWrapper *pMndWrapper; SMgmtWrapper *pWrapper; } SMsgHandle; @@ -146,14 +176,11 @@ void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); void dndReleaseWrapper(SMgmtWrapper *pWrapper); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); // dndMonitor.h void dndSendMonitorReport(SDnode *pDnode); -// dndMsg.h -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); - // dndStr.h const char *dndStatStr(EDndStatus stat); const char *dndNodeLogStr(ENodeType ntype); @@ -168,12 +195,12 @@ SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); int32_t dndInitMsgHandle(SDnode *pDnode); // mgmt -void dmGetMgmtFp(SMgmtWrapper *pWrapper); -void bmGetMgmtFp(SMgmtWrapper *pWrapper); -void qmGetMgmtFp(SMgmtWrapper *pMgmt); -void smGetMgmtFp(SMgmtWrapper *pWrapper); -void vmGetMgmtFp(SMgmtWrapper *pWrapper); -void mmGetMgmtFp(SMgmtWrapper *pMgmt); +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void bmSetMgmtFp(SMgmtWrapper *pWrapper); +void qmSetMgmtFp(SMgmtWrapper *pMgmt); +void smSetMgmtFp(SMgmtWrapper *pWrapper); +void vmSetMgmtFp(SMgmtWrapper *pWrapper); +void mmSetMgmtFp(SMgmtWrapper *pMgmt); void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); @@ -194,8 +221,8 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - + SMonGrantInfo *pGrantInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 72feed7c62..80fefdb3ef 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -82,12 +82,12 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { } dndSetStatus(pDnode, DND_STAT_INIT); - dmGetMgmtFp(&pDnode->wrappers[DNODE]); - mmGetMgmtFp(&pDnode->wrappers[MNODE]); - vmGetMgmtFp(&pDnode->wrappers[VNODES]); - qmGetMgmtFp(&pDnode->wrappers[QNODE]); - smGetMgmtFp(&pDnode->wrappers[SNODE]); - bmGetMgmtFp(&pDnode->wrappers[BNODE]); + dmSetMgmtFp(&pDnode->wrappers[DNODE]); + mmSetMgmtFp(&pDnode->wrappers[MNODE]); + vmSetMgmtFp(&pDnode->wrappers[VNODES]); + qmSetMgmtFp(&pDnode->wrappers[QNODE]); + smSetMgmtFp(&pDnode->wrappers[SNODE]); + bmSetMgmtFp(&pDnode->wrappers[BNODE]); for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/src/mmInt.c index bd7446780d..97fdafe139 100644 --- a/source/dnode/mgmt/mm/src/mmInt.c +++ b/source/dnode/mgmt/mm/src/mmInt.c @@ -227,7 +227,7 @@ static int32_t mmStart(SMgmtWrapper *pWrapper) { return mndStart(pMgmt->pMnode); } -void mmGetMgmtFp(SMgmtWrapper *pWrapper) { +void mmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = mmOpen; mgmtFp.closeFp = mmClose; diff --git a/source/dnode/mgmt/qm/src/qmInt.c b/source/dnode/mgmt/qm/src/qmInt.c index 8079859b96..53c43bf841 100644 --- a/source/dnode/mgmt/qm/src/qmInt.c +++ b/source/dnode/mgmt/qm/src/qmInt.c @@ -112,7 +112,7 @@ int32_t qmOpen(SMgmtWrapper *pWrapper) { return code; } -void qmGetMgmtFp(SMgmtWrapper *pWrapper) { +void qmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = qmOpen; mgmtFp.closeFp = qmClose; diff --git a/source/dnode/mgmt/sm/src/smInt.c b/source/dnode/mgmt/sm/src/smInt.c index 6d2e2aaefd..a279655609 100644 --- a/source/dnode/mgmt/sm/src/smInt.c +++ b/source/dnode/mgmt/sm/src/smInt.c @@ -109,7 +109,7 @@ int32_t smOpen(SMgmtWrapper *pWrapper) { return code; } -void smGetMgmtFp(SMgmtWrapper *pWrapper) { +void smSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = smOpen; mgmtFp.closeFp = smClose; diff --git a/source/dnode/mgmt/vm/src/vmInt.c b/source/dnode/mgmt/vm/src/vmInt.c index d9ef7b5ae6..d78a567b69 100644 --- a/source/dnode/mgmt/vm/src/vmInt.c +++ b/source/dnode/mgmt/vm/src/vmInt.c @@ -333,7 +333,7 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { return 0; } -void vmGetMgmtFp(SMgmtWrapper *pWrapper) { +void vmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = vmInit; mgmtFp.closeFp = vmCleanup; From e41043074bff1643c73ed98d70caf05782152749 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Apr 2022 08:20:26 -0400 Subject: [PATCH 27/32] integrate constant calculate --- include/libs/function/functionMgt.h | 1 + source/libs/function/src/functionMgt.c | 4 ++++ source/libs/nodes/src/nodesCodeFuncs.c | 5 +++-- source/libs/parser/src/parCalcConst.c | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 7d46b543cb..207e936801 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -127,6 +127,7 @@ bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); +bool fmIsPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 4034a0eb0f..5bd7f22987 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -104,6 +104,10 @@ bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsPseudoColumnFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); +} + bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e39a7ecfa6..1b16ba9206 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1491,6 +1491,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkValueGenByCalc = "GenByCalc"; static const char* jkValueLiteral = "Literal"; static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; @@ -1544,7 +1545,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkValueTranslate, pNode->genByCalc); + code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc); } if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); @@ -1617,7 +1618,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->genByCalc); + code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc); } if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index a572ccf5bd..46cdcc4243 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "functionMgt.h" #include "parInt.h" #include "scalar.h" @@ -44,6 +45,9 @@ static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { SFunctionNode* pFunc = *pNode; + if (fmIsPseudoColumnFunc(pFunc->funcId)) { + return DEAL_RES_CONTINUE; + } SNode* pParam = NULL; FOREACH(pParam, pFunc->pParameterList) { if (QUERY_NODE_VALUE != nodeType(pParam)) { From 1839793b143b0c81844d0e15a595b46a2d41af2a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 20:31:11 +0800 Subject: [PATCH 28/32] rename files --- source/dnode/mgmt/CMakeLists.txt | 26 +++++++------------ source/dnode/mgmt/bm/{src => }/bmHandle.c | 0 source/dnode/mgmt/bm/{src => }/bmInt.c | 0 source/dnode/mgmt/bm/{src => }/bmWorker.c | 0 source/dnode/mgmt/dm/{src => }/dmFile.c | 0 source/dnode/mgmt/dm/{src => }/dmHandle.c | 0 source/dnode/mgmt/dm/{src => }/dmInt.c | 0 source/dnode/mgmt/dm/{src => }/dmWorker.c | 0 source/dnode/mgmt/{main => }/exe/dndMain.c | 0 source/dnode/mgmt/{bm => }/inc/bmInt.h | 0 source/dnode/mgmt/{dm => }/inc/dmInt.h | 0 source/dnode/mgmt/{main => }/inc/dnd.h | 0 source/dnode/mgmt/{main => }/inc/dndInt.h | 0 source/dnode/mgmt/{mm => }/inc/mmInt.h | 0 source/dnode/mgmt/{qm => }/inc/qmInt.h | 0 source/dnode/mgmt/{sm => }/inc/smInt.h | 0 source/dnode/mgmt/{vm => }/inc/vmInt.h | 0 source/dnode/mgmt/main/{src => }/dndEnv.c | 0 source/dnode/mgmt/main/{src => }/dndExec.c | 0 source/dnode/mgmt/main/{src => }/dndFile.c | 0 source/dnode/mgmt/main/{src => }/dndInt.c | 0 source/dnode/mgmt/main/{src => }/dndMonitor.c | 0 .../dnode/mgmt/main/{src => }/dndTransport.c | 0 source/dnode/mgmt/mm/{src => }/mmFile.c | 0 source/dnode/mgmt/mm/{src => }/mmHandle.c | 0 source/dnode/mgmt/mm/{src => }/mmInt.c | 0 source/dnode/mgmt/mm/{src => }/mmWorker.c | 0 source/dnode/mgmt/qm/{src => }/qmHandle.c | 0 source/dnode/mgmt/qm/{src => }/qmInt.c | 0 source/dnode/mgmt/qm/{src => }/qmWorker.c | 0 source/dnode/mgmt/sm/{src => }/smHandle.c | 0 source/dnode/mgmt/sm/{src => }/smInt.c | 0 source/dnode/mgmt/sm/{src => }/smWorker.c | 0 source/dnode/mgmt/vm/{src => }/vmFile.c | 0 source/dnode/mgmt/vm/{src => }/vmHandle.c | 0 source/dnode/mgmt/vm/{src => }/vmInt.c | 0 source/dnode/mgmt/vm/{src => }/vmWorker.c | 0 37 files changed, 10 insertions(+), 16 deletions(-) rename source/dnode/mgmt/bm/{src => }/bmHandle.c (100%) rename source/dnode/mgmt/bm/{src => }/bmInt.c (100%) rename source/dnode/mgmt/bm/{src => }/bmWorker.c (100%) rename source/dnode/mgmt/dm/{src => }/dmFile.c (100%) rename source/dnode/mgmt/dm/{src => }/dmHandle.c (100%) rename source/dnode/mgmt/dm/{src => }/dmInt.c (100%) rename source/dnode/mgmt/dm/{src => }/dmWorker.c (100%) rename source/dnode/mgmt/{main => }/exe/dndMain.c (100%) rename source/dnode/mgmt/{bm => }/inc/bmInt.h (100%) rename source/dnode/mgmt/{dm => }/inc/dmInt.h (100%) rename source/dnode/mgmt/{main => }/inc/dnd.h (100%) rename source/dnode/mgmt/{main => }/inc/dndInt.h (100%) rename source/dnode/mgmt/{mm => }/inc/mmInt.h (100%) rename source/dnode/mgmt/{qm => }/inc/qmInt.h (100%) rename source/dnode/mgmt/{sm => }/inc/smInt.h (100%) rename source/dnode/mgmt/{vm => }/inc/vmInt.h (100%) rename source/dnode/mgmt/main/{src => }/dndEnv.c (100%) rename source/dnode/mgmt/main/{src => }/dndExec.c (100%) rename source/dnode/mgmt/main/{src => }/dndFile.c (100%) rename source/dnode/mgmt/main/{src => }/dndInt.c (100%) rename source/dnode/mgmt/main/{src => }/dndMonitor.c (100%) rename source/dnode/mgmt/main/{src => }/dndTransport.c (100%) rename source/dnode/mgmt/mm/{src => }/mmFile.c (100%) rename source/dnode/mgmt/mm/{src => }/mmHandle.c (100%) rename source/dnode/mgmt/mm/{src => }/mmInt.c (100%) rename source/dnode/mgmt/mm/{src => }/mmWorker.c (100%) rename source/dnode/mgmt/qm/{src => }/qmHandle.c (100%) rename source/dnode/mgmt/qm/{src => }/qmInt.c (100%) rename source/dnode/mgmt/qm/{src => }/qmWorker.c (100%) rename source/dnode/mgmt/sm/{src => }/smHandle.c (100%) rename source/dnode/mgmt/sm/{src => }/smInt.c (100%) rename source/dnode/mgmt/sm/{src => }/smWorker.c (100%) rename source/dnode/mgmt/vm/{src => }/vmFile.c (100%) rename source/dnode/mgmt/vm/{src => }/vmHandle.c (100%) rename source/dnode/mgmt/vm/{src => }/vmInt.c (100%) rename source/dnode/mgmt/vm/{src => }/vmWorker.c (100%) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 81c434eb27..0c9a3733a2 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,10 +1,10 @@ -aux_source_directory(dm/src DNODE_SRC) -aux_source_directory(qm/src DNODE_SRC) -aux_source_directory(bm/src DNODE_SRC) -aux_source_directory(sm/src DNODE_SRC) -aux_source_directory(vm/src DNODE_SRC) -aux_source_directory(mm/src DNODE_SRC) -aux_source_directory(main/src DNODE_SRC) +aux_source_directory(dm DNODE_SRC) +aux_source_directory(qm DNODE_SRC) +aux_source_directory(bm DNODE_SRC) +aux_source_directory(sm DNODE_SRC) +aux_source_directory(vm DNODE_SRC) +aux_source_directory(mm DNODE_SRC) +aux_source_directory(main DNODE_SRC) add_library(dnode STATIC ${DNODE_SRC}) target_link_libraries( dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor @@ -12,20 +12,14 @@ target_link_libraries( target_include_directories( dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -aux_source_directory(main/exe EXEC_SRC) +aux_source_directory(exe EXEC_SRC) add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/bm/src/bmHandle.c b/source/dnode/mgmt/bm/bmHandle.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmHandle.c rename to source/dnode/mgmt/bm/bmHandle.c diff --git a/source/dnode/mgmt/bm/src/bmInt.c b/source/dnode/mgmt/bm/bmInt.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmInt.c rename to source/dnode/mgmt/bm/bmInt.c diff --git a/source/dnode/mgmt/bm/src/bmWorker.c b/source/dnode/mgmt/bm/bmWorker.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmWorker.c rename to source/dnode/mgmt/bm/bmWorker.c diff --git a/source/dnode/mgmt/dm/src/dmFile.c b/source/dnode/mgmt/dm/dmFile.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmFile.c rename to source/dnode/mgmt/dm/dmFile.c diff --git a/source/dnode/mgmt/dm/src/dmHandle.c b/source/dnode/mgmt/dm/dmHandle.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmHandle.c rename to source/dnode/mgmt/dm/dmHandle.c diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/dmInt.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmInt.c rename to source/dnode/mgmt/dm/dmInt.c diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/dmWorker.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmWorker.c rename to source/dnode/mgmt/dm/dmWorker.c diff --git a/source/dnode/mgmt/main/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c similarity index 100% rename from source/dnode/mgmt/main/exe/dndMain.c rename to source/dnode/mgmt/exe/dndMain.c diff --git a/source/dnode/mgmt/bm/inc/bmInt.h b/source/dnode/mgmt/inc/bmInt.h similarity index 100% rename from source/dnode/mgmt/bm/inc/bmInt.h rename to source/dnode/mgmt/inc/bmInt.h diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/inc/dmInt.h similarity index 100% rename from source/dnode/mgmt/dm/inc/dmInt.h rename to source/dnode/mgmt/inc/dmInt.h diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/inc/dnd.h similarity index 100% rename from source/dnode/mgmt/main/inc/dnd.h rename to source/dnode/mgmt/inc/dnd.h diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/inc/dndInt.h similarity index 100% rename from source/dnode/mgmt/main/inc/dndInt.h rename to source/dnode/mgmt/inc/dndInt.h diff --git a/source/dnode/mgmt/mm/inc/mmInt.h b/source/dnode/mgmt/inc/mmInt.h similarity index 100% rename from source/dnode/mgmt/mm/inc/mmInt.h rename to source/dnode/mgmt/inc/mmInt.h diff --git a/source/dnode/mgmt/qm/inc/qmInt.h b/source/dnode/mgmt/inc/qmInt.h similarity index 100% rename from source/dnode/mgmt/qm/inc/qmInt.h rename to source/dnode/mgmt/inc/qmInt.h diff --git a/source/dnode/mgmt/sm/inc/smInt.h b/source/dnode/mgmt/inc/smInt.h similarity index 100% rename from source/dnode/mgmt/sm/inc/smInt.h rename to source/dnode/mgmt/inc/smInt.h diff --git a/source/dnode/mgmt/vm/inc/vmInt.h b/source/dnode/mgmt/inc/vmInt.h similarity index 100% rename from source/dnode/mgmt/vm/inc/vmInt.h rename to source/dnode/mgmt/inc/vmInt.h diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/dndEnv.c similarity index 100% rename from source/dnode/mgmt/main/src/dndEnv.c rename to source/dnode/mgmt/main/dndEnv.c diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/dndExec.c similarity index 100% rename from source/dnode/mgmt/main/src/dndExec.c rename to source/dnode/mgmt/main/dndExec.c diff --git a/source/dnode/mgmt/main/src/dndFile.c b/source/dnode/mgmt/main/dndFile.c similarity index 100% rename from source/dnode/mgmt/main/src/dndFile.c rename to source/dnode/mgmt/main/dndFile.c diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/dndInt.c similarity index 100% rename from source/dnode/mgmt/main/src/dndInt.c rename to source/dnode/mgmt/main/dndInt.c diff --git a/source/dnode/mgmt/main/src/dndMonitor.c b/source/dnode/mgmt/main/dndMonitor.c similarity index 100% rename from source/dnode/mgmt/main/src/dndMonitor.c rename to source/dnode/mgmt/main/dndMonitor.c diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/dndTransport.c similarity index 100% rename from source/dnode/mgmt/main/src/dndTransport.c rename to source/dnode/mgmt/main/dndTransport.c diff --git a/source/dnode/mgmt/mm/src/mmFile.c b/source/dnode/mgmt/mm/mmFile.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmFile.c rename to source/dnode/mgmt/mm/mmFile.c diff --git a/source/dnode/mgmt/mm/src/mmHandle.c b/source/dnode/mgmt/mm/mmHandle.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmHandle.c rename to source/dnode/mgmt/mm/mmHandle.c diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/mmInt.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmInt.c rename to source/dnode/mgmt/mm/mmInt.c diff --git a/source/dnode/mgmt/mm/src/mmWorker.c b/source/dnode/mgmt/mm/mmWorker.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmWorker.c rename to source/dnode/mgmt/mm/mmWorker.c diff --git a/source/dnode/mgmt/qm/src/qmHandle.c b/source/dnode/mgmt/qm/qmHandle.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmHandle.c rename to source/dnode/mgmt/qm/qmHandle.c diff --git a/source/dnode/mgmt/qm/src/qmInt.c b/source/dnode/mgmt/qm/qmInt.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmInt.c rename to source/dnode/mgmt/qm/qmInt.c diff --git a/source/dnode/mgmt/qm/src/qmWorker.c b/source/dnode/mgmt/qm/qmWorker.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmWorker.c rename to source/dnode/mgmt/qm/qmWorker.c diff --git a/source/dnode/mgmt/sm/src/smHandle.c b/source/dnode/mgmt/sm/smHandle.c similarity index 100% rename from source/dnode/mgmt/sm/src/smHandle.c rename to source/dnode/mgmt/sm/smHandle.c diff --git a/source/dnode/mgmt/sm/src/smInt.c b/source/dnode/mgmt/sm/smInt.c similarity index 100% rename from source/dnode/mgmt/sm/src/smInt.c rename to source/dnode/mgmt/sm/smInt.c diff --git a/source/dnode/mgmt/sm/src/smWorker.c b/source/dnode/mgmt/sm/smWorker.c similarity index 100% rename from source/dnode/mgmt/sm/src/smWorker.c rename to source/dnode/mgmt/sm/smWorker.c diff --git a/source/dnode/mgmt/vm/src/vmFile.c b/source/dnode/mgmt/vm/vmFile.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmFile.c rename to source/dnode/mgmt/vm/vmFile.c diff --git a/source/dnode/mgmt/vm/src/vmHandle.c b/source/dnode/mgmt/vm/vmHandle.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmHandle.c rename to source/dnode/mgmt/vm/vmHandle.c diff --git a/source/dnode/mgmt/vm/src/vmInt.c b/source/dnode/mgmt/vm/vmInt.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmInt.c rename to source/dnode/mgmt/vm/vmInt.c diff --git a/source/dnode/mgmt/vm/src/vmWorker.c b/source/dnode/mgmt/vm/vmWorker.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmWorker.c rename to source/dnode/mgmt/vm/vmWorker.c From e4404dc8e72bb8574ac83d61714e42ba029463d2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 21:02:58 +0800 Subject: [PATCH 29/32] add more log --- source/client/src/tmq.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 81219e1c36..6c0bdd4bab 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -140,6 +140,7 @@ typedef struct { tmq_t* tmq; SMqClientVg* pVg; int32_t epoch; + int32_t vgId; tsem_t rspSem; tmq_message_t** msg; int32_t sync; @@ -839,7 +840,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { - tscWarn("msg discard, code:%x", code); + tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code); goto CREATE_MSG_FAIL; } @@ -848,12 +849,12 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (msgEpoch < tmqEpoch) { /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ /*tsem_post(&tmq->rspSem);*/ - tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); + tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); return 0; } if (msgEpoch != tmqEpoch) { - tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); + tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); } else { atomic_sub_fetch_32(&tmq->waitingRequest, 1); } @@ -1041,7 +1042,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { if (epStatus == 1) { int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); - if (epSkipCnt < 40) return 0; + if (epSkipCnt < 5000) return 0; } atomic_store_32(&tmq->epSkipCnt, 0); int32_t tlen = sizeof(SMqCMGetSubEpReq); @@ -1256,6 +1257,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { } pParam->tmq = tmq; pParam->pVg = pVg; + pParam->vgId = pVg->vgId; pParam->epoch = tmq->epoch; pParam->sync = 0; @@ -1282,7 +1284,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); - tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset); + tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; From ac82b91ed17759cdc5ef819e8877ffae0d32a3e2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 23:00:12 +0800 Subject: [PATCH 30/32] handle except --- source/libs/transport/src/transSrv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index dfb5eb35d6..79c72b3a35 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -193,6 +193,7 @@ static void uvHandleReq(SSrvConn* pConn) { transMsg.ahandle = (void*)pHead->ahandle; transMsg.handle = NULL; + // transDestroyBuffer(&pConn->readBuf); transClearBuffer(&pConn->readBuf); pConn->inType = pHead->msgType; if (pConn->status == ConnNormal) { @@ -249,6 +250,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { conn->broken = true; if (conn->status == ConnAcquire) { if (conn->regArg.init) { + tTrace("server conn %p broken, notify server app", conn); STrans* pTransInst = conn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); memset(&conn->regArg, 0, sizeof(conn->regArg)); @@ -270,7 +272,7 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnSendCb(uv_write_t* req, int status) { SSrvConn* conn = req->data; - transClearBuffer(&conn->readBuf); + // transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { From 798c3a9f825239103f572b5ca0e2bbe07a59b75f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 23:03:12 +0800 Subject: [PATCH 31/32] add more log --- source/client/src/tmq.c | 12 ++++++----- source/dnode/mnode/impl/src/mndConsumer.c | 4 +--- source/dnode/mnode/impl/src/mndScheduler.c | 3 +++ source/dnode/mnode/impl/src/mndSubscribe.c | 24 ++++++++++++++++------ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6c0bdd4bab..602ecdf6ab 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -917,10 +917,10 @@ CREATE_MSG_FAIL: bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ - tscDebug("consumer %ld update ep epoch %d to epoch %d", tmq->consumerId, tmq->epoch, epoch); bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { return false; @@ -938,17 +938,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { taosHashClear(pHash); topic.topicName = strdup(pTopicEp->topic); + tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int32_t j = 0; j < topicNumCur; j++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); if (vgNumCur == 0) break; for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer %ld epoch %d vg %d build %s\n", tmq->consumerId, epoch, pVgCur->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -962,10 +964,10 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("consumer %ld epoch %d vg %d offset og to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("consumer %ld epoch %d vg %d found %s\n", tmq->consumerId, epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); } tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { @@ -1231,7 +1233,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1); - tscDebug("consumer %ld skip vg %d skip cnt %ld", tmq->consumerId, pVg->vgId, skipCnt); + tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt); continue; #if 0 if (skipCnt < 30000) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index c4c93b7fc9..6080ec7710 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -160,10 +160,8 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); - pOldConsumer->epoch++; - - // TODO handle update /*taosWLockLatch(&pOldConsumer->lock);*/ + atomic_add_fetch_32(&pOldConsumer->epoch, 1); /*taosWUnLockLatch(&pOldConsumer->lock);*/ return 0; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 4562d9e5d3..df7946a0c1 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -471,6 +471,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib consumerEp.consumerId = -1; consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; + + mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId); + int32_t msgLen; if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { sdbRelease(pSdb, pVgroup); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index ae9a4198a4..57d98396ea 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -212,19 +212,24 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } + //TODO add lock ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + int32_t serverEpoch = pConsumer->epoch; // TODO int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mTrace("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, pConsumer->epoch, hbStatus); + mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, hbStatus); atomic_store_32(&pConsumer->hbStatus, 0); /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ strcpy(rsp.cgroup, pReq->cgroup); - if (epoch != pConsumer->epoch) { - mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch); + if (epoch != serverEpoch) { + mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, serverEpoch); + mDebug("consumer %ld try r lock", consumerId); + taosRLockLatch(&pConsumer->lock); + mDebug("consumer %ld r locked", consumerId); SArray *pTopics = pConsumer->currentTopics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -238,7 +243,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); if (consumerId == pSubConsumer->consumerId) { int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - mInfo("topic %s has %d vg", topicName, pConsumer->epoch); + mInfo("topic %s has %d vg", topicName, serverEpoch); SMqSubTopicEp topicEp; strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); @@ -264,6 +269,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { } mndReleaseSubscribe(pMnode, pSub); } + taosRUnLockLatch(&pConsumer->lock); + mDebug("consumer %ld r unlock", consumerId); } int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); @@ -272,7 +279,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { return -1; } ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = pConsumer->epoch; + ((SMqRspHead *)buf)->epoch = serverEpoch; ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -395,7 +402,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); taosMemoryFreeClear(pRebSub->key); - mInfo("mq rebalance subscription: %s", pSub->key); + mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, (int32_t)taosArrayGetSize(pSub->unassignedVg)); // remove lost consumer for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { @@ -442,6 +449,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { } SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); + mDebug("consumer %ld try w lock", pRebConsumer->consumerId); + taosWLockLatch(&pRebConsumer->lock); + mDebug("consumer %ld w locked", pRebConsumer->consumerId); int32_t status = atomic_load_32(&pRebConsumer->status); if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || @@ -462,6 +472,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); mndTransAppendCommitlog(pTrans, pConsumerRaw); } + taosWUnLockLatch(&pRebConsumer->lock); + mDebug("consumer %ld w unlock", pRebConsumer->consumerId); mndReleaseConsumer(pMnode, pRebConsumer); } From 5298436e1f67e193d3169d5745f9eb285cda903a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 23:20:11 +0800 Subject: [PATCH 32/32] add test for multi vgroup consume --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b756976394..5a3ee003f0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -41,7 +41,7 @@ # ---- tmq ./test.sh -f tsim/tmq/basic.sim ./test.sh -f tsim/tmq/basic1.sim -#./test.sh -f tsim/tmq/oneTopic.sim +./test.sh -f tsim/tmq/oneTopic.sim #./test.sh -f tsim/tmq/multiTopic.sim # --- stable