From 87fefa9e15f638a0a950f4a05322c320995f1f44 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 01/42] [TD-14241]: add string functions --- include/libs/function/functionMgt.h | 9 +- include/libs/scalar/scalar.h | 13 + source/libs/function/src/builtins.c | 82 +++++- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/sclfunc.c | 408 ++++++++++++++++++++++++---- 5 files changed, 453 insertions(+), 61 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 7d46b543cb..0969fb203a 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -72,10 +72,15 @@ typedef enum EFunctionType { FUNCTION_TYPE_ATAN, // string function - FUNCTION_TYPE_CHAR_LENGTH = 1500, + FUNCTION_TYPE_LENGTH = 1500, + FUNCTION_TYPE_CHAR_LENGTH, FUNCTION_TYPE_CONCAT, FUNCTION_TYPE_CONCAT_WS, - FUNCTION_TYPE_LENGTH, + FUNCTION_TYPE_LOWER, + FUNCTION_TYPE_UPPER, + FUNCTION_TYPE_LTRIM, + FUNCTION_TYPE_RTRIM, + FUNCTION_TYPE_SUBSTR, // conversion function FUNCTION_TYPE_CAST = 2000, diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index c6d17ef65c..10ef0c1b23 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -42,6 +42,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); int32_t vectorGetConvertType(int32_t type1, int32_t type2); int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut); +/* Math functions */ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); @@ -58,6 +59,18 @@ int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +/* String functions */ +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); + + #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0a5a90d2d9..4c07ba924d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -282,6 +282,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = atanFunction, .finalizeFunc = NULL }, + { + .name = "length", + .type = FUNCTION_TYPE_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lengthFunction, + .finalizeFunc = NULL + }, + { + .name = "char_length", + .type = FUNCTION_TYPE_CHAR_LENGTH, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = charLengthFunction, + .finalizeFunc = NULL + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, @@ -289,7 +309,67 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .checkFunc = stubCheckAndGetResultType, .getEnvFunc = NULL, .initFunc = NULL, - .sprocessFunc = NULL, + .sprocessFunc = concatFunction, + .finalizeFunc = NULL + }, + { + .name = "concat_ws", + .type = FUNCTION_TYPE_CONCAT_WS, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = concatWsFunction, + .finalizeFunc = NULL + }, + { + .name = "lower", + .type = FUNCTION_TYPE_LOWER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = lowerFunction, + .finalizeFunc = NULL + }, + { + .name = "upper", + .type = FUNCTION_TYPE_UPPER, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = upperFunction, + .finalizeFunc = NULL + }, + { + .name = "ltrim", + .type = FUNCTION_TYPE_LTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = ltrimFunction, + .finalizeFunc = NULL + }, + { + .name = "rtrim", + .type = FUNCTION_TYPE_RTRIM, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = rtrimFunction, + .finalizeFunc = NULL + }, + { + .name = "substr", + .type = FUNCTION_TYPE_SUBSTR, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_STRING_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = substrFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index cf34fc24a9..58b62bb05e 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -47,7 +47,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); #define GET_PARAM_TYPE(_c) ((_c)->columnData->info.type) -#define GET_PARAM_BYTES(_c) ((_c)->pColumnInfoData->info.bytes) +#define GET_PARAM_BYTES(_c) ((_c)->columnData->info.bytes) void sclFreeParam(SScalarParam *param); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index af5f07751b..15a9984c80 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -3,10 +3,14 @@ #include "sclInt.h" #include "sclvector.h" -static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { -// dst->type = src->type; -// dst->bytes = src->bytes; -// dst->num = src->num; +typedef float (*_float_fn)(float); +typedef double (*_double_fn)(double); +typedef double (*_double_fn_2)(double, double); +typedef int (*_conv_fn)(int); +typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); + +double tlog(double v, double base) { + return log(v) / log(base); } /** Math functions **/ @@ -107,14 +111,6 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu return TSDB_CODE_SUCCESS; } -typedef float (*_float_fn)(float); -typedef double (*_double_fn)(double); -typedef double (*_double_fn_2)(double, double); - -double tlog(double v, double base) { - return log(v) / log(base); -} - int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SScalarParam* pOutput, _double_fn valFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_NUMERIC_TYPE(type)) { @@ -216,6 +212,341 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p return TSDB_CODE_SUCCESS; } +/** String functions **/ +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char **in = (char **)pInputData->pData; + int16_t *out = (int16_t *)pOutputData->pData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + out[i] = varDataLen(in[i]); + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + char **in = (char **)pInputData->pData; + int16_t *out = (int16_t *)pOutputData->pData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + if (type == TSDB_DATA_TYPE_VARCHAR) { + out[i] = varDataLen(in[i]); + } else { //NCHAR + out[i] = varDataLen(in[i]) / TSDB_NCHAR_SIZE; + } + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings + return TSDB_CODE_FAILED; + } + + SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } + pInputData[i] = pInput[i].columnData; + } + + bool hasNull = false; + for (int32_t k = 0; k < pInput->numOfRows; ++k) { + for (int32_t i = 0; i < inputNum; ++i) { + if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { + colDataSetNull_f(pOutputData->nullbitmap, k); + hasNull = true; + break; + } + } + + if (hasNull) { + continue; + } + + char *in = NULL; + char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); + + int16_t dataLen = 0; + for (int32_t i = 0; i < inputNum; ++i) { + in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); + + memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); + dataLen += varDataLen(in); + } + varDataSetLen(out, dataLen); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(pInputData); + + return TSDB_CODE_SUCCESS; +} + +int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum < 3 || inputNum > 9) { // concat accpet 3-9 input strings including the separator + return TSDB_CODE_FAILED; + } + + SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } + pInputData[i] = pInput[i].columnData; + } + + for (int32_t k = 0; k < pInput->numOfRows; ++k) { + char *sep = pInputData[0]->pData; + if (colDataIsNull_f(pInputData[0]->nullbitmap, k)) { + colDataSetNull_f(pOutputData->nullbitmap, k); + continue; + } + + char *in = NULL; + char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); + + int16_t dataLen = 0; + for (int32_t i = 1; i < inputNum; ++i) { + if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { + continue; + } + + in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); + memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); + dataLen += varDataLen(in); + + if (i < inputNum - 1) { + //insert the separator + memcpy(varDataVal(out) + dataLen, varDataVal(sep), varDataLen(sep)); + dataLen += varDataLen(sep); + } + } + varDataSetLen(out, dataLen); + } + + pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(pInputData); + + return TSDB_CODE_SUCCESS; +} + +int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); + char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); + + int32_t len = varDataLen(in); + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t j = 0; j < len; ++j) { + *(varDataVal(out) + j) = convFn(*(varDataVal(in) + j)); + } + } else { //NCHAR + for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) { + *((uint32_t *)varDataVal(out) + j) = convFn(*((uint32_t *)varDataVal(in) + j)); + } + } + varDataSetLen(out, len); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + +void tltrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = 0; i < charLen; ++i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = 0; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); + } + + varDataSetLen(output, resLen); +} + +void trtrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = charLen - 1; i >= 0; --i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = charLen - 1; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + } + memcpy(varDataVal(output), varDataVal(input), resLen); + + varDataSetLen(output, resLen); +} + +int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { + int32_t type = GET_PARAM_TYPE(pInput); + if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { + return TSDB_CODE_FAILED; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); + char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); + + int32_t len = varDataLen(in); + int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; + trimFn(in, out, type, charLen); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + +int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + if (inputNum != 2 || inputNum!= 3) { + return TSDB_CODE_FAILED; + } + + int32_t subPos = 0; + GET_TYPED_DATA(subPos, int32_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + if (subPos == 0) { //subPos needs to be positive or negative values; + return TSDB_CODE_FAILED; + } + + int32_t subLen = INT16_MAX; + if (inputNum == 3) { + GET_TYPED_DATA(subLen, int32_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + if (subLen < 0) { //subLen cannot be negative + return TSDB_CODE_FAILED; + } + subLen = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subLen : subLen * TSDB_NCHAR_SIZE; + } + + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + for (int32_t i = 0; i < pOutput->numOfRows; ++i) { + if (colDataIsNull_f(pInputData->nullbitmap, i)) { + colDataSetNull_f(pOutputData->nullbitmap, i); + continue; + } + + char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); + char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); + + int32_t len = varDataLen(in); + int32_t startPosBytes; + + if (subPos > 0) { + startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE; + startPosBytes = MIN(startPosBytes, len); + } else { + startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE; + startPosBytes = MAX(startPosBytes, 0); + } + + subLen = MIN(subLen, len - startPosBytes); + if (subLen > 0) { + memcpy(varDataVal(out), varDataVal(in) + startPosBytes, subLen); + } + + varDataSetLen(out, subLen); + } + + pOutput->numOfRows = pInput->numOfRows; + + return TSDB_CODE_SUCCESS; +} + + int32_t atanFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return doScalarFunctionUnique(pInput, inputNum, pOutput, atan); } @@ -264,57 +595,20 @@ int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return doScalarFunction(pInput, inputNum, pOutput, roundf, round); } -static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assert(numOfInput == 1); -#if 0 - int64_t* out = (int64_t*) pOutput->data; - char* s = pLeft->data; - - for(int32_t i = 0; i < pLeft->num; ++i) { - out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); - } -#endif +int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doCaseConvFunction(pInput, inputNum, pOutput, tolower); } -static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - assert(numOfInput > 0); -#if 0 - int32_t rowLen = 0; - int32_t num = 1; - for(int32_t i = 0; i < numOfInput; ++i) { - rowLen += pLeft[i].bytes; - - if (pLeft[i].num > 1) { - num = pLeft[i].num; - } - } - - pOutput->data = taosMemoryRealloc(pOutput->data, rowLen * num); - assert(pOutput->data); - - char* rstart = pOutput->data; - for(int32_t i = 0; i < num; ++i) { - - char* s = rstart; - varDataSetLen(s, 0); - for (int32_t j = 0; j < numOfInput; ++j) { - char* p1 = POINTER_SHIFT(pLeft[j].data, i * pLeft[j].bytes); - - memcpy(varDataVal(s) + varDataLen(s), varDataVal(p1), varDataLen(p1)); - varDataLen(s) += varDataLen(p1); - } - - rstart += rowLen; - } -#endif +int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doCaseConvFunction(pInput, inputNum, pOutput, toupper); } -static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - +int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doTrimFunction(pInput, inputNum, pOutput, tltrim); } -static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { - +int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doTrimFunction(pInput, inputNum, pOutput, trtrim); } static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { From b0eefba1b3fa738c8a29e087c76beb0e80dce796 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 02/42] [TD-14241]: add string functions --- source/libs/scalar/src/sclfunc.c | 169 +++++++++++++++---------------- 1 file changed, 81 insertions(+), 88 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 15a9984c80..12eee76838 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -8,12 +8,13 @@ typedef double (*_double_fn)(double); typedef double (*_double_fn_2)(double, double); typedef int (*_conv_fn)(int); typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); +typedef int16_t (*_len_fn)(char *, int32_t); +/** Math functions **/ double tlog(double v, double base) { return log(v) / log(base); } -/** Math functions **/ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; @@ -213,32 +214,78 @@ int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam* p } /** String functions **/ -int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - int32_t type = GET_PARAM_TYPE(pInput); - if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { - return TSDB_CODE_FAILED; - } - - SColumnInfoData *pInputData = pInput->columnData; - SColumnInfoData *pOutputData = pOutput->columnData; - - char **in = (char **)pInputData->pData; - int16_t *out = (int16_t *)pOutputData->pData; - - for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); - continue; - } - - out[i] = varDataLen(in[i]); - } - - pOutput->numOfRows = pInput->numOfRows; - return TSDB_CODE_SUCCESS; +int16_t tlength(char *input, int32_t type) { + return varDataLen(input); } -int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +int16_t tcharlength(char *input, int32_t type) { + if (type == TSDB_DATA_TYPE_VARCHAR) { + return varDataLen(input); + } else { //NCHAR + return varDataLen(input) / TSDB_NCHAR_SIZE; + } +} + +void tltrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = 0; i < charLen; ++i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = 0; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); + } + + varDataSetLen(output, resLen); +} + +void trtrim(char *input, char *output, int32_t type, int32_t charLen) { + int32_t numOfSpaces = 0; + if (type == TSDB_DATA_TYPE_VARCHAR) { + for (int32_t i = charLen - 1; i >= 0; --i) { + if (!isspace(*(varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } else { //NCHAR + for (int32_t i = charLen - 1; i < charLen; ++i) { + if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { + break; + } + numOfSpaces++; + } + } + + int32_t resLen; + if (type == TSDB_DATA_TYPE_VARCHAR) { + resLen = charLen - numOfSpaces; + } else { + resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; + } + memcpy(varDataVal(output), varDataVal(input), resLen); + + varDataSetLen(output, resLen); +} + +int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _len_fn lenFn) { int32_t type = GET_PARAM_TYPE(pInput); if (inputNum != 1 || !IS_VAR_DATA_TYPE(type)) { return TSDB_CODE_FAILED; @@ -256,11 +303,7 @@ int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam continue; } - if (type == TSDB_DATA_TYPE_VARCHAR) { - out[i] = varDataLen(in[i]); - } else { //NCHAR - out[i] = varDataLen(in[i]) / TSDB_NCHAR_SIZE; - } + out[i] = lenFn(in[i], type); } pOutput->numOfRows = pInput->numOfRows; @@ -403,64 +446,6 @@ int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return TSDB_CODE_SUCCESS; } -void tltrim(char *input, char *output, int32_t type, int32_t charLen) { - int32_t numOfSpaces = 0; - if (type == TSDB_DATA_TYPE_VARCHAR) { - for (int32_t i = 0; i < charLen; ++i) { - if (!isspace(*(varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } else { //NCHAR - for (int32_t i = 0; i < charLen; ++i) { - if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } - - int32_t resLen; - if (type == TSDB_DATA_TYPE_VARCHAR) { - resLen = charLen - numOfSpaces; - memcpy(varDataVal(output), varDataVal(input) + numOfSpaces, resLen); - } else { - resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; - memcpy(varDataVal(output), varDataVal(input) + numOfSpaces * TSDB_NCHAR_SIZE, resLen); - } - - varDataSetLen(output, resLen); -} - -void trtrim(char *input, char *output, int32_t type, int32_t charLen) { - int32_t numOfSpaces = 0; - if (type == TSDB_DATA_TYPE_VARCHAR) { - for (int32_t i = charLen - 1; i >= 0; --i) { - if (!isspace(*(varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } else { //NCHAR - for (int32_t i = charLen - 1; i < charLen; ++i) { - if (!iswspace(*((uint32_t *)varDataVal(input) + i))) { - break; - } - numOfSpaces++; - } - } - - int32_t resLen; - if (type == TSDB_DATA_TYPE_VARCHAR) { - resLen = charLen - numOfSpaces; - } else { - resLen = (charLen - numOfSpaces) * TSDB_NCHAR_SIZE; - } - memcpy(varDataVal(output), varDataVal(input), resLen); - - varDataSetLen(output, resLen); -} int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _trim_fn trimFn) { int32_t type = GET_PARAM_TYPE(pInput); @@ -611,6 +596,14 @@ int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut return doTrimFunction(pInput, inputNum, pOutput, trtrim); } +int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doLengthFunction(pInput, inputNum, pOutput, tlength); +} + +int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return doLengthFunction(pInput, inputNum, pOutput, tcharlength); +} + static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: From 89abd4e3495d2ce733a05ef59d6b17ca165fcf12 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 03/42] [TD-14241]: add string functions --- source/libs/function/src/builtins.c | 39 +++++++++++++++++++++-------- source/libs/scalar/src/sclfunc.c | 5 ++-- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4c07ba924d..a18386aff6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -478,16 +478,6 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; break; } - case FUNCTION_TYPE_CONCAT: - case FUNCTION_TYPE_ROWTS: - case FUNCTION_TYPE_TBNAME: - case FUNCTION_TYPE_QSTARTTS: - case FUNCTION_TYPE_QENDTS: - case FUNCTION_TYPE_WSTARTTS: - case FUNCTION_TYPE_WENDTS: - case FUNCTION_TYPE_WDURATION: - // todo - break; case FUNCTION_TYPE_ABS: case FUNCTION_TYPE_CEIL: @@ -512,6 +502,35 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } + case FUNCTION_TYPE_LENGTH: + case FUNCTION_TYPE_CHAR_LENGTH: { + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_SMALLINT].bytes, .type = TSDB_DATA_TYPE_SMALLINT }; + break; + } + + case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT_WS: + case FUNCTION_TYPE_LOWER: + case FUNCTION_TYPE_UPPER: + case FUNCTION_TYPE_LTRIM: + case FUNCTION_TYPE_RTRIM: + case FUNCTION_TYPE_SUBSTR: { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); + int32_t paraType = pParam->node.resType.type; + pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + break; + } + + case FUNCTION_TYPE_ROWTS: + case FUNCTION_TYPE_TBNAME: + case FUNCTION_TYPE_QSTARTTS: + case FUNCTION_TYPE_QENDTS: + case FUNCTION_TYPE_WSTARTTS: + case FUNCTION_TYPE_WENDTS: + case FUNCTION_TYPE_WDURATION: + // todo + break; + default: ASSERT(0); // to found the fault ASAP. } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 12eee76838..8eee608ee0 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -294,7 +294,7 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char **in = (char **)pInputData->pData; + char *in = pInputData->pData; int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -303,7 +303,8 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p continue; } - out[i] = lenFn(in[i], type); + out[i] = lenFn(in, type); + in += varDataTLen(in); } pOutput->numOfRows = pInput->numOfRows; From 89aba48960cc6afd5c342cb61e15e078d6fcc074 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 04/42] [TD-14241]: concat function adoption --- source/libs/scalar/src/sclfunc.c | 39 ++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 8eee608ee0..864b8acf75 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -310,6 +310,8 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } +void allocateOutputBuf() { +} int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { if (inputNum < 2 || inputNum > 8) { // concat accpet 2-8 input strings @@ -318,17 +320,35 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; + char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); + char *output = NULL; + int32_t inputLen = 0; + int32_t numOfRows = pInput->numOfRows; for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { return TSDB_CODE_FAILED; } pInputData[i] = pInput[i].columnData; + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + input[i] = pInputData[i]->pData; } + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; + pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); + pOutputData->info.type = GET_PARAM_TYPE(pInput); + pOutputData->info.bytes = outputLen; + pOutputData->varmeta.length = outputLen; + pOutputData->varmeta.allocLen = outputLen; + } + output = pOutputData->pData; + bool hasNull = false; - for (int32_t k = 0; k < pInput->numOfRows; ++k) { + int32_t offset = 0; + for (int32_t k = 0; k < numOfRows; ++k) { for (int32_t i = 0; i < inputNum; ++i) { if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { colDataSetNull_f(pOutputData->nullbitmap, k); @@ -341,20 +361,21 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } - char *in = NULL; - char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); - int16_t dataLen = 0; for (int32_t i = 0; i < inputNum; ++i) { - in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); - - memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); - dataLen += varDataLen(in); + memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); + dataLen += varDataLen(input[i]); + input[i] += varDataTLen(input[i]); } - varDataSetLen(out, dataLen); + varDataSetLen(output, dataLen); + int32_t dataTLen = varDataTLen(output); + output += dataTLen; + pOutputData->varmeta.offset[k] = offset; + offset += dataTLen; } pOutput->numOfRows = pInput->numOfRows; + taosMemoryFree(input); taosMemoryFree(pInputData); return TSDB_CODE_SUCCESS; From 6f6d0e4a0997d2c592e07fefcab16d8829c6c608 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 05/42] [TD-14241]: concat function adoption --- source/libs/function/src/builtins.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a18386aff6..a19acb096d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -508,7 +508,22 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } - case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT: { + int32_t paraTypeFirst, totalBytes = 0; + for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { + SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + int32_t paraType = pParam->node.resType.type; + if (i == 0) { + paraTypeFirst = paraType; + } + if (paraType != paraTypeFirst) { + return TSDB_CODE_FAILED; + } + totalBytes += pParam->node.resType.bytes; + } + pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; + break; + } case FUNCTION_TYPE_CONCAT_WS: case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: @@ -517,7 +532,8 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_SUBSTR: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); int32_t paraType = pParam->node.resType.type; - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + //pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; + pFunc->node.resType = (SDataType) { .bytes = 23, .type = paraType }; break; } From 6eb6cfe279b214bc0e457bd5f2fbd24576746255 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 06/42] [TD-14241]: concat_ws function adoption --- source/libs/function/src/builtins.c | 18 ++++++++--- source/libs/scalar/src/sclfunc.c | 47 ++++++++++++++++++++++------- 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a19acb096d..4cbc91fa6a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -508,12 +508,19 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { break; } - case FUNCTION_TYPE_CONCAT: { - int32_t paraTypeFirst, totalBytes = 0; - for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { + case FUNCTION_TYPE_CONCAT: + case FUNCTION_TYPE_CONCAT_WS: { + int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; + int32_t firstParamIndex = 0; + if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { + firstParamIndex = 1; + SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); + sepBytes = pSep->node.resType.type; + } + for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); int32_t paraType = pParam->node.resType.type; - if (i == 0) { + if (i == firstParamIndex) { paraTypeFirst = paraType; } if (paraType != paraTypeFirst) { @@ -521,10 +528,11 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { } totalBytes += pParam->node.resType.bytes; } + //TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. + totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; break; } - case FUNCTION_TYPE_CONCAT_WS: case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: case FUNCTION_TYPE_LTRIM: diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 864b8acf75..f1eab5c05c 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -374,7 +374,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu offset += dataTLen; } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; taosMemoryFree(input); taosMemoryFree(pInputData); @@ -388,45 +388,70 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; + char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); + char *output = NULL; + int32_t inputLen = 0; + int32_t numOfRows = pInput[1].numOfRows; for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { return TSDB_CODE_FAILED; } pInputData[i] = pInput[i].columnData; + if (i == 0) { + // calculate required separator space + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * (inputNum - 2); + } else { + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + } + input[i] = pInputData[i]->pData; } - for (int32_t k = 0; k < pInput->numOfRows; ++k) { + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; + pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); + pOutputData->info.type = GET_PARAM_TYPE(pInput); + pOutputData->info.bytes = outputLen; + pOutputData->varmeta.length = outputLen; + pOutputData->varmeta.allocLen = outputLen; + } + output = pOutputData->pData; + + int32_t offset = 0; + for (int32_t k = 0; k < numOfRows; ++k) { char *sep = pInputData[0]->pData; if (colDataIsNull_f(pInputData[0]->nullbitmap, k)) { colDataSetNull_f(pOutputData->nullbitmap, k); continue; } - char *in = NULL; - char *out = pOutputData->pData + k * GET_PARAM_BYTES(pOutput); - int16_t dataLen = 0; for (int32_t i = 1; i < inputNum; ++i) { if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { continue; } - in = pInputData[i]->pData + k * GET_PARAM_BYTES(&pInput[i]); - memcpy(varDataVal(out) + dataLen, varDataVal(in), varDataLen(in)); - dataLen += varDataLen(in); + memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); + dataLen += varDataLen(input[i]); + input[i] += varDataTLen(input[i]); if (i < inputNum - 1) { //insert the separator - memcpy(varDataVal(out) + dataLen, varDataVal(sep), varDataLen(sep)); + memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep)); dataLen += varDataLen(sep); } } - varDataSetLen(out, dataLen); + varDataSetLen(output, dataLen); + int32_t dataTLen = varDataTLen(output); + output += dataTLen; + pOutputData->varmeta.offset[k] = offset; + offset += dataTLen; } - pOutput->numOfRows = pInput->numOfRows; + pOutput->numOfRows = numOfRows; + taosMemoryFree(input); taosMemoryFree(pInputData); return TSDB_CODE_SUCCESS; From dddf621aa7a4504f756c1d2aa96a18a417b8a85c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 07/42] [TD-14241]: concat/concat_ws fix input has both constant and column issue --- source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 52 +++++++++++++++++++---------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4cbc91fa6a..957e7ee406 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -526,6 +526,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { if (paraType != paraTypeFirst) { return TSDB_CODE_FAILED; } + //TODO: for constants also needs numOfRows totalBytes += pParam->node.resType.bytes; } //TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index f1eab5c05c..21d8aee0ec 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -310,7 +310,13 @@ int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } -void allocateOutputBuf() { + +static void setVarTypeOutputBuf(SColumnInfoData *pOutputData, int32_t len, int32_t type) { + pOutputData->pData = taosMemoryCalloc(len, sizeof(char)); + pOutputData->info.type = type; + pOutputData->info.bytes = len; + pOutputData->varmeta.length = len; + pOutputData->varmeta.allocLen = len; } int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { @@ -324,25 +330,30 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu char *output = NULL; int32_t inputLen = 0; - int32_t numOfRows = pInput->numOfRows; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; + } + } for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { return TSDB_CODE_FAILED; } pInputData[i] = pInput[i].columnData; - inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; input[i] = pInputData[i]->pData; + if (pInput[i].numOfRows == 1) { + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; + } else { + inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; + } } //allocate output buf if (pOutputData->pData == NULL) { int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; - pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); - pOutputData->info.type = GET_PARAM_TYPE(pInput); - pOutputData->info.bytes = outputLen; - pOutputData->varmeta.length = outputLen; - pOutputData->varmeta.allocLen = outputLen; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); } output = pOutputData->pData; @@ -365,7 +376,9 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu for (int32_t i = 0; i < inputNum; ++i) { memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); dataLen += varDataLen(input[i]); - input[i] += varDataTLen(input[i]); + if (pInput[i].numOfRows != 1) { + input[i] += varDataTLen(input[i]); + } } varDataSetLen(output, dataLen); int32_t dataTLen = varDataTLen(output); @@ -392,7 +405,12 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *output = NULL; int32_t inputLen = 0; - int32_t numOfRows = pInput[1].numOfRows; + int32_t numOfRows = 0; + for (int32_t i = 0; i < inputNum; ++i) { + if (pInput[i].numOfRows > numOfRows) { + numOfRows = pInput[i].numOfRows; + } + } for (int32_t i = 0; i < inputNum; ++i) { if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { @@ -401,7 +419,9 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p pInputData[i] = pInput[i].columnData; if (i == 0) { // calculate required separator space - inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * (inputNum - 2); + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2); + } else if (pInput[i].numOfRows == 1) { + inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; } else { inputLen += pInputData[i]->varmeta.length - numOfRows * VARSTR_HEADER_SIZE; } @@ -411,11 +431,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p //allocate output buf if (pOutputData->pData == NULL) { int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; - pOutputData->pData = taosMemoryCalloc(outputLen, sizeof(char)); - pOutputData->info.type = GET_PARAM_TYPE(pInput); - pOutputData->info.bytes = outputLen; - pOutputData->varmeta.length = outputLen; - pOutputData->varmeta.allocLen = outputLen; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(&pInput[1])); } output = pOutputData->pData; @@ -435,7 +451,9 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p memcpy(varDataVal(output) + dataLen, varDataVal(input[i]), varDataLen(input[i])); dataLen += varDataLen(input[i]); - input[i] += varDataTLen(input[i]); + if (pInput[i].numOfRows != 1) { + input[i] += varDataTLen(input[i]); + } if (i < inputNum - 1) { //insert the separator From c846c6ebf30ba307e5ba028253294a6637de0b25 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 08/42] [TD-14241]: lower/upper function adoption --- source/libs/function/src/builtins.c | 6 +++--- source/libs/scalar/src/sclfunc.c | 26 +++++++++++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 957e7ee406..dd9e32655f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -540,9 +540,9 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_RTRIM: case FUNCTION_TYPE_SUBSTR: { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, 0); - int32_t paraType = pParam->node.resType.type; - //pFunc->node.resType = (SDataType) { .bytes = tDataTypes[paraType].bytes, .type = paraType }; - pFunc->node.resType = (SDataType) { .bytes = 23, .type = paraType }; + int32_t paraType = pParam->node.resType.type; + int32_t paraBytes = pParam->node.resType.bytes; + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; break; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 21d8aee0ec..e6eb3c43b4 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -484,26 +484,38 @@ int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = pInputData->varmeta.length; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); + } + + char *input = pInputData->pData; + char *output = pOutputData->pData; + + int32_t offset = 0; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_f(pInputData->nullbitmap, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } - char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); - char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); - - int32_t len = varDataLen(in); + int32_t len = varDataLen(input); if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t j = 0; j < len; ++j) { - *(varDataVal(out) + j) = convFn(*(varDataVal(in) + j)); + *(varDataVal(output) + j) = convFn(*(varDataVal(input) + j)); } } else { //NCHAR for (int32_t j = 0; j < len / TSDB_NCHAR_SIZE; ++j) { - *((uint32_t *)varDataVal(out) + j) = convFn(*((uint32_t *)varDataVal(in) + j)); + *((uint32_t *)varDataVal(output) + j) = convFn(*((uint32_t *)varDataVal(input) + j)); } } - varDataSetLen(out, len); + varDataSetLen(output, len); + input += varDataTLen(input); + output += varDataTLen(output); + + pOutputData->varmeta.offset[i] = offset; + offset += len + VARSTR_HEADER_SIZE; } pOutput->numOfRows = pInput->numOfRows; From 64184611d7e35c1aa84e59a91640e6b380e6b811 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 09/42] [TD-14241]: ltrim/rtrim function adoption --- source/libs/scalar/src/sclfunc.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e6eb3c43b4..9f481c8fb2 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -533,18 +533,32 @@ int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = pInputData->varmeta.length; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); + } + + char *input = pInputData->pData; + char *output = pOutputData->pData; + + int32_t offset = 0; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_f(pInputData->nullbitmap, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } - char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); - char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); - - int32_t len = varDataLen(in); + int32_t len = varDataLen(input); int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; - trimFn(in, out, type, charLen); + trimFn(input, output, type, charLen); + + varDataSetLen(output, len); + input += varDataTLen(input); + output += varDataTLen(output); + + pOutputData->varmeta.offset[i] = offset; + offset += len + VARSTR_HEADER_SIZE; } pOutput->numOfRows = pInput->numOfRows; From 5a1548bf1548b9a9f66d8ea7283037703482ae13 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 30 Mar 2022 14:03:54 +0800 Subject: [PATCH 10/42] [TD-14241]: substr function adoption --- source/libs/function/src/builtins.c | 49 ++++++++++++++++------------- source/libs/scalar/src/sclfunc.c | 30 ++++++++++++------ 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index dd9e32655f..dc43a3508f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -510,29 +510,36 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { case FUNCTION_TYPE_CONCAT: case FUNCTION_TYPE_CONCAT_WS: { - int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; - int32_t firstParamIndex = 0; - if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { - firstParamIndex = 1; - SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); - sepBytes = pSep->node.resType.type; - } - for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { + int32_t paraType, paraBytes = 0; + for (int32_t i = 0; i < pFunc->pParameterList->length; ++i) { SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); - int32_t paraType = pParam->node.resType.type; - if (i == firstParamIndex) { - paraTypeFirst = paraType; - } - if (paraType != paraTypeFirst) { - return TSDB_CODE_FAILED; - } - //TODO: for constants also needs numOfRows - totalBytes += pParam->node.resType.bytes; + paraBytes += pParam->node.resType.bytes; + paraType = pParam->node.resType.type; } - //TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. - totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; - pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; - break; + pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; + //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; + //int32_t firstParamIndex = 0; + //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { + // firstParamIndex = 1; + // SColumnNode* pSep = nodesListGetNode(pFunc->pParameterList, 0); + // sepBytes = pSep->node.resType.type; + //} + //for (int32_t i = firstParamIndex; i < pFunc->pParameterList->length; ++i) { + // SColumnNode* pParam = nodesListGetNode(pFunc->pParameterList, i); + // int32_t paraType = pParam->node.resType.type; + // if (i == firstParamIndex) { + // paraTypeFirst = paraType; + // } + // if (paraType != paraTypeFirst) { + // return TSDB_CODE_FAILED; + // } + // //TODO: for constants also needs numOfRows + // totalBytes += pParam->node.resType.bytes; + //} + ////TODO: need to get numOfRows to decide how much space separator needed. Currently set to 100. + //totalBytes += sepBytes * (pFunc->pParameterList->length - 2) * 100; + //pFunc->node.resType = (SDataType) { .bytes = totalBytes, .type = paraTypeFirst }; + //break; } case FUNCTION_TYPE_LOWER: case FUNCTION_TYPE_UPPER: diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 9f481c8fb2..222f7ac69a 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -350,7 +350,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } } - //allocate output buf + allocate output buf if (pOutputData->pData == NULL) { int32_t outputLen = inputLen + numOfRows * VARSTR_HEADER_SIZE; setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); @@ -567,7 +567,7 @@ int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 2 || inputNum!= 3) { + if (inputNum != 2 && inputNum!= 3) { return TSDB_CODE_FAILED; } @@ -589,16 +589,23 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - for (int32_t i = 0; i < pOutput->numOfRows; ++i) { + //allocate output buf + if (pOutputData->pData == NULL) { + int32_t outputLen = pInputData->varmeta.length; + setVarTypeOutputBuf(pOutputData, outputLen, GET_PARAM_TYPE(pInput)); + } + + char *input = pInputData->pData; + char *output = pOutputData->pData; + + int32_t offset = 0; + for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_f(pInputData->nullbitmap, i)) { colDataSetNull_f(pOutputData->nullbitmap, i); continue; } - char *in = pInputData->pData + i * GET_PARAM_BYTES(pInput); - char *out = pOutputData->pData + i * GET_PARAM_BYTES(pInput); - - int32_t len = varDataLen(in); + int32_t len = varDataLen(input); int32_t startPosBytes; if (subPos > 0) { @@ -611,10 +618,15 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu subLen = MIN(subLen, len - startPosBytes); if (subLen > 0) { - memcpy(varDataVal(out), varDataVal(in) + startPosBytes, subLen); + memcpy(varDataVal(output), varDataVal(input) + startPosBytes, subLen); } - varDataSetLen(out, subLen); + varDataSetLen(output, subLen); + input += varDataTLen(input); + output += varDataTLen(output); + + pOutputData->varmeta.offset[i] = offset; + offset += subLen + VARSTR_HEADER_SIZE; } pOutput->numOfRows = pInput->numOfRows; From a8a914d47df5a2e98d743d2ae213bb8625b8ba3f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 13:39:01 +0800 Subject: [PATCH 11/42] minor changes --- source/dnode/mgmt/main/inc/dnd.h | 35 +++++++++++++++++---------- source/dnode/mgmt/main/inc/dndInt.h | 20 +++++++--------- source/dnode/mgmt/main/src/dndEnv.c | 37 ----------------------------- source/dnode/mgmt/main/src/dndInt.c | 16 ++++++++++++- source/dnode/mgmt/main/src/dndMsg.c | 29 +++++++++++++++++++--- 5 files changed, 72 insertions(+), 65 deletions(-) diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index 255a739add..f3a409f257 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -134,33 +134,42 @@ typedef struct SDnode { SMgmtWrapper wrappers[NODE_MAX]; } SDnode; +// dndFile.h +int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); +int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); + +// dndInt.h +EDndStatus dndGetStatus(SDnode *pDnode); +void dndSetStatus(SDnode *pDnode, EDndStatus stat); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); +SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); +int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); +void dndReleaseWrapper(SMgmtWrapper *pWrapper); + +// dndMonitor.h +void dndSendMonitorReport(SDnode *pDnode); + +// dndMsg.h +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); + +// dndStr.h +const char *dndStatStr(EDndStatus stat); const char *dndNodeLogStr(ENodeType ntype); const char *dndNodeProcStr(ENodeType ntype); const char *dndEventStr(EDndEvent ev); -EDndStatus dndGetStatus(SDnode *pDnode); -void dndSetStatus(SDnode *pDnode, EDndStatus stat); -void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -void dndSendMonitorReport(SDnode *pDnode); +// dndTransport.h int32_t dndInitServer(SDnode *pDnode); void dndCleanupServer(SDnode *pDnode); int32_t dndInitClient(SDnode *pDnode); void dndCleanupClient(SDnode *pDnode); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); -int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); -int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); - -SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); -void dndReleaseWrapper(SMgmtWrapper *pWrapper); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 612d35d513..0c4ada1df3 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -29,26 +29,24 @@ extern "C" { #endif -// dndInt.c -int32_t dndInit(); -void dndCleanup(); -const char *dndStatStr(EDndStatus stat); -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); +// dndEnv.h +int32_t dndInit(); +void dndCleanup(); -// dndMsg.c -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); - -// dndExec.c +// dndExec.h int32_t dndOpenNode(SMgmtWrapper *pWrapper); void dndCloseNode(SMgmtWrapper *pWrapper); int32_t dndRun(SDnode *pDnode); -// dndObj.c +// dndInt.c SDnode *dndCreate(const SDnodeOpt *pOption); void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); +// dndMsg.c +void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); + // dndTransport.c int32_t dndInitMsgHandle(SDnode *pDnode); void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/src/dndEnv.c index 8792147822..9f75594335 100644 --- a/source/dnode/mgmt/main/src/dndEnv.c +++ b/source/dnode/mgmt/main/src/dndEnv.c @@ -57,40 +57,3 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } - -void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { - pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; - pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; -} - -EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } - -void dndSetStatus(SDnode *pDnode, EDndStatus status) { - if (pDnode->status != status) { - dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); - pDnode->status = status; - } -} - -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); -} diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 6f6e21b983..25e4d89c5b 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -188,4 +188,18 @@ void dndReleaseWrapper(SMgmtWrapper *pWrapper) { int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); taosRUnLockLatch(&pWrapper->latch); dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount); -} \ No newline at end of file +} + +void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { + pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; + pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; +} + +EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } + +void dndSetStatus(SDnode *pDnode, EDndStatus status) { + if (pDnode->status != status) { + dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status)); + pDnode->status = status; + } +} diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index 2ab1e401c7..bb0f8ccb89 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -66,8 +66,8 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); code = (*msgFp)(pWrapper, pMsg); } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, - pRpc->ahandle, pMsg->user); + dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, + pMsg->user); code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); } else { dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, @@ -171,4 +171,27 @@ int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; return -1; } -} \ No newline at end of file +} + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} From 48312d911558b882c78e9fde03a7430a7e694efa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 15:23:08 +0800 Subject: [PATCH 12/42] handle except --- source/libs/index/src/indexFst.c | 3 +- source/libs/index/src/log | 1445 +++++++++++++++++++++++++++++ source/libs/index/test/fstTest.cc | 73 +- 3 files changed, 1514 insertions(+), 7 deletions(-) create mode 100644 source/libs/index/src/log diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 24bc7a93a2..a146564c9e 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -1251,7 +1251,6 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); - fstNodeDestroy(node); } else { // This is a little tricky. We're in this case if the // given bound is not a prefix of any key in the FST. @@ -1349,7 +1348,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } - FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); + FstSlice slice = fstSliceCreate(buf, isz); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); diff --git a/source/libs/index/src/log b/source/libs/index/src/log new file mode 100644 index 0000000000..0b6cb8ddc2 --- /dev/null +++ b/source/libs/index/src/log @@ -0,0 +1,1445 @@ +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 1) /* +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 2) * Copyright (c) 2019 TAOS Data, Inc. +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 3) * +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 4) * This program is free software: you can use, redistribute, and/or modify +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 5) * it under the terms of the GNU Affero General Public License, version 3 +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 6) * or later ("AGPL"), as published by the Free Software Foundation. +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 7) * +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 8) * This program is distributed in the hope that it will be useful, but WITHOUT +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 9) * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 10) * FITNESS FOR A PARTICULAR PURPOSE. +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 11) * +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 12) * You should have received a copy of the GNU Affero General Public License +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 13) * along with this program. If not, see . +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 14) */ +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 15) +5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 16) #include "indexFst.h" +5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 17) #include "indexFstAutomation.h" +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 18) #include "indexInt.h" +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 19) #include "tchecksum.h" +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 20) #include "tcoding.h" +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 21) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 22) static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 23) CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 24) fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 25) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 26) static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { +20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 27) uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 28) fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 29) return nBytes; +20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 30) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 31) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 32) FstUnFinishedNodes* fstUnFinishedNodesCreate() { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 33) FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 34) if (nodes == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 35) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 36) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 37) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 38) nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 39) fstUnFinishedNodesPushEmpty(nodes, false); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 40) return nodes; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 41) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 42) static void unFinishedNodeDestroyElem(void* elem) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 43) FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 44) fstBuilderNodeDestroy(b->node); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 45) taosMemoryFree(b->last); +7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 46) b->last = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 47) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 48) void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 49) if (nodes == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 50) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 51) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 52) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 53) taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 54) taosMemoryFree(nodes); +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 55) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 56) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 57) void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 58) FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 59) node->isFinal = isFinal; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 60) node->finalOutput = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 61) node->trans = taosArrayInit(16, sizeof(FstTransition)); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 62) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 63) FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 64) taosArrayPush(nodes->stack, &un); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 65) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 66) FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 67) assert(taosArrayGetSize(nodes->stack) == 1); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 68) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 69) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 70) assert(un->last == NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 71) return un->node; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 72) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 73) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 74) FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 75) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 76) fstBuilderNodeUnfinishedLastCompiled(un, addr); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 77) // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 78) // un->last = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 79) return un->node; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 80) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 81) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 82) FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 83) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 84) assert(un->last == NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 85) return un->node; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 86) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 87) void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 88) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 89) un->node->isFinal = true; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 90) un->node->finalOutput = out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 91) // un->node->trans = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 92) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 93) void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 94) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 95) fstBuilderNodeUnfinishedLastCompiled(un, addr); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 96) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 97) void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 98) FstSlice* s = &bs; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 99) if (fstSliceIsEmpty(s)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 100) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 101) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 102) size_t sz = taosArrayGetSize(nodes->stack) - 1; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 103) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 104) assert(un->last == NULL); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 105) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 106) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 107) // trn->inp = s->data[s->start]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 108) // trn->out = out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 109) int32_t len = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 110) uint8_t* data = fstSliceData(s, &len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 111) un->last = fstLastTransitionCreate(data[0], out); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 112) +d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 113) for (uint64_t i = 1; i < len; i++) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 114) FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 115) n->isFinal = false; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 116) n->finalOutput = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 117) n->trans = taosArrayInit(16, sizeof(FstTransition)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 118) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 119) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 120) // trn->inp = s->data[i]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 121) // trn->out = out; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 122) FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 123) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 124) FstBuilderNodeUnfinished un = {.node = n, .last = trn}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 125) taosArrayPush(nodes->stack, &un); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 126) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 127) fstUnFinishedNodesPushEmpty(nodes, true); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 128) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 129) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 130) uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 131) FstSlice* s = &bs; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 132) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 133) size_t ssz = taosArrayGetSize(node->stack); // stack size +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 134) uint64_t count = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 135) int32_t lsz; // data len +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 136) uint8_t* data = fstSliceData(s, &lsz); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 137) for (size_t i = 0; i < ssz && i < lsz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 138) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 139) if (un->last->inp == data[i]) { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 140) count++; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 141) } else { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 142) break; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 143) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 144) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 145) return count; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 146) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 147) uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 148) FstSlice* s = &bs; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 149) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 150) size_t lsz = (size_t)(s->end - s->start + 1); // data len +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 151) size_t ssz = taosArrayGetSize(node->stack); // stack size +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 152) *out = in; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 153) uint64_t i = 0; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 154) for (i = 0; i < lsz && i < ssz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 155) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 156) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 157) FstLastTransition* t = un->last; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 158) uint64_t addPrefix = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 159) uint8_t* data = fstSliceData(s, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 160) if (t && t->inp == data[i]) { +92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 161) uint64_t commPrefix = TMIN(t->out, *out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 162) uint64_t tAddPrefix = t->out - commPrefix; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 163) (*out) = (*out) - commPrefix; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 164) t->out = commPrefix; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 165) addPrefix = tAddPrefix; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 166) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 167) break; +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 168) } +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 169) if (addPrefix != 0) { +73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 170) if (i + 1 < ssz) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 171) FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 172) fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); +73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 173) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 174) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 175) } +16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 176) return i; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 177) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 178) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 179) FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { +93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 180) FstState fs = {.state = EmptyFinal, .val = 0}; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 181) if (addr == EMPTY_ADDRESS) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 182) return fs; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 183) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 184) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 185) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 186) uint8_t v = data[addr]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 187) uint8_t t = (v & 0b11000000) >> 6; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 188) if (t == 0b11) { +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 189) fs.state = OneTransNext; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 190) } else if (t == 0b10) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 191) fs.state = OneTrans; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 192) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 193) fs.state = AnyTrans; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 194) } +93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 195) fs.val = v; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 196) return fs; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 197) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 198) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 199) static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 200) {.state = OneTrans, .val = 0b10000000}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 201) {.state = AnyTrans, .val = 0b00000000}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 202) {.state = EmptyFinal, .val = 0b00000000}}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 203) // debug +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 204) static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 205) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 206) FstState fstStateCreate(State state) { +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 207) uint8_t idx = (uint8_t)state; +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 208) return fstStateDict[idx]; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 209) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 210) // compile +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 211) void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 212) FstState s = fstStateCreate(OneTransNext); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 213) fstStateSetCommInput(&s, inp); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 214) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 215) bool null = false; +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 216) uint8_t v = fstStateCommInput(&s, &null); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 217) if (null) { +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 218) // w->write_all(&[inp]) +7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 219) fstCountingWriterWrite(w, &inp, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 220) } +7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 221) fstCountingWriterWrite(w, &(s.val), 1); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 222) // w->write_all(&[s.val]) +7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 223) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 224) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 225) void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 226) Output out = trn->out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 227) uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 228) uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 229) PackSizes packSizes = 0; +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 230) +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 231) FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 232) FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 233) fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 234) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 235) FstState st = fstStateCreate(OneTrans); +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 236) +d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 237) fstStateSetCommInput(&st, trn->inp); +22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 238) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 239) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 240) uint8_t inp = fstStateCommInput(&st, &null); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 241) if (null == true) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 242) fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 243) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 244) fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 245) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 246) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 247) void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 248) size_t sz = taosArrayGetSize(node->trans); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 249) assert(sz <= 256); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 250) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 251) uint8_t tSize = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 252) uint8_t oSize = packSize(node->finalOutput); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 253) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 254) // finalOutput.is_zero() +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 255) bool anyOuts = (node->finalOutput != 0); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 256) for (size_t i = 0; i < sz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 257) FstTransition* t = taosArrayGet(node->trans, i); +92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 258) tSize = TMAX(tSize, packDeltaSize(addr, t->addr)); +92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 259) oSize = TMAX(oSize, packSize(t->out)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 260) anyOuts = anyOuts || (t->out != 0); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 261) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 262) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 263) PackSizes packSizes = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 264) if (anyOuts) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 265) FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 266) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 267) FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 268) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 269) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 270) FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 271) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 272) FstState st = fstStateCreate(AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 273) fstStateSetFinalState(&st, node->isFinal); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 274) fstStateSetStateNtrans(&st, (uint8_t)sz); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 275) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 276) if (anyOuts) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 277) if (FST_BUILDER_NODE_IS_FINAL(node)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 278) fstCountingWriterPackUintIn(w, node->finalOutput, oSize); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 279) } +d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 280) for (int32_t i = sz - 1; i >= 0; i--) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 281) FstTransition* t = taosArrayGet(node->trans, i); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 282) fstCountingWriterPackUintIn(w, t->out, oSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 283) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 284) } +d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 285) for (int32_t i = sz - 1; i >= 0; i--) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 286) FstTransition* t = taosArrayGet(node->trans, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 287) fstPackDeltaIn(w, addr, t->addr, tSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 288) } +d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 289) for (int32_t i = sz - 1; i >= 0; i--) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 290) FstTransition* t = taosArrayGet(node->trans, i); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 291) fstCountingWriterWrite(w, (char*)&t->inp, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 292) // fstPackDeltaIn(w, addr, t->addr, tSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 293) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 294) if (sz > TRANS_INDEX_THRESHOLD) { +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 295) // A value of 255 indicates that no transition exists for the byte +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 296) // at that index. (Except when there are 256 transitions.) Namely, +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 297) // any value greater than or equal to the number of transitions in +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 298) // this node indicates an absent transition. +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 299) uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); +984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 300) memset(index, 255, sizeof(uint8_t) * 256); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 301) /// for (uint8_t i = 0; i < 256; i++) { +984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 302) // index[i] = 255; +984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 303) ///} +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 304) for (size_t i = 0; i < sz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 305) FstTransition* t = taosArrayGet(node->trans, i); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 306) index[t->inp] = i; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 307) // fstPackDeltaIn(w, addr, t->addr, tSize); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 308) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 309) fstCountingWriterWrite(w, (char*)index, 256); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 310) taosMemoryFree(index); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 311) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 312) fstCountingWriterWrite(w, (char*)&packSizes, 1); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 313) bool null = false; +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 314) fstStateStateNtrans(&st, &null); +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 315) if (null == true) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 316) // 256 can't be represented in a u8, so we abuse the fact that +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 317) // the # of transitions can never be 1 here, since 1 is always +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 318) // encoded in the state byte. +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 319) uint8_t v = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 320) if (sz == 256) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 321) fstCountingWriterWrite(w, (char*)&v, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 322) } else { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 323) fstCountingWriterWrite(w, (char*)&sz, 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 324) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 325) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 326) fstCountingWriterWrite(w, (char*)(&(st.val)), 1); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 327) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 328) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 329) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 330) // set_comm_input +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 331) void fstStateSetCommInput(FstState* s, uint8_t inp) { +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 332) assert(s->state == OneTransNext || s->state == OneTrans); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 333) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 334) uint8_t val; +9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 335) COMMON_INDEX(inp, 0b111111, val); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 336) s->val = (s->val & fstStateDict[s->state].val) | val; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 337) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 338) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 339) // comm_input +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 340) uint8_t fstStateCommInput(FstState* s, bool* null) { +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 341) assert(s->state == OneTransNext || s->state == OneTrans); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 342) uint8_t v = s->val & 0b00111111; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 343) if (v == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 344) *null = true; +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 345) return v; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 346) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 347) // v = 0 indicate that common_input is None +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 348) return v == 0 ? 0 : COMMON_INPUT(v); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 349) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 350) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 351) // input_len +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 352) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 353) uint64_t fstStateInputLen(FstState* s) { +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 354) assert(s->state == OneTransNext || s->state == OneTrans); +d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 355) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 356) fstStateCommInput(s, &null); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 357) return null ? 1 : 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 358) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 359) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 360) // end_addr +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 361) uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { +4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 362) assert(s->state == OneTransNext); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 363) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 364) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 365) uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { +4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 366) assert(s->state == OneTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 367) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 368) - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 369) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 370) uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 371) uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 372) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 373) uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 374) return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 375) - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 376) - finalOsize; // final output +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 377) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 378) // input +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 379) uint8_t fstStateInput(FstState* s, FstNode* node) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 380) assert(s->state == OneTransNext || s->state == OneTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 381) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 382) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 383) uint8_t inp = fstStateCommInput(s, &null); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 384) uint8_t* data = fstSliceData(slice, NULL); +9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 385) return null == false ? inp : data[node->start - 1]; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 386) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 387) uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 388) assert(s->state == AnyTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 389) FstSlice* slice = &node->data; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 390) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 391) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 392) - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 393) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 394) uint8_t* data = fstSliceData(slice, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 395) return data[at]; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 396) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 397) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 398) // trans_addr +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 399) CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 400) assert(s->state == OneTransNext || s->state == OneTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 401) FstSlice* slice = &node->data; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 402) if (s->state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 403) return (CompiledAddr)(node->end) - 1; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 404) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 405) PackSizes sizes = node->sizes; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 406) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 407) uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 408) - tSizes; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 409) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 410) // refactor error logic +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 411) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 412) return unpackDelta(data + i, tSizes, node->end); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 413) } +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 414) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 415) CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 416) assert(s->state == AnyTrans); +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 417) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 418) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 419) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 420) uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 421) node->nTrans - (i * tSizes) - tSizes; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 422) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 423) return unpackDelta(data + at, tSizes, node->end); +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 424) } +0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 425) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 426) // sizes +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 427) PackSizes fstStateSizes(FstState* s, FstSlice* slice) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 428) assert(s->state == OneTrans || s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 429) uint64_t i; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 430) if (s->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 431) i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 432) } else { +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 433) i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 434) } +169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 435) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 436) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 437) return (PackSizes)(*(data + i)); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 438) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 439) // Output +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 440) Output fstStateOutput(FstState* s, FstNode* node) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 441) assert(s->state == OneTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 442) +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 443) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 444) if (oSizes == 0) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 445) return 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 446) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 447) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 448) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 449) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 450) uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 451) uint8_t* data = fstSliceData(slice, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 452) return unpackUint64(data + i, oSizes); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 453) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 454) Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 455) assert(s->state == AnyTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 456) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 457) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 458) if (oSizes == 0) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 459) return 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 460) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 461) FstSlice* slice = &node->data; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 462) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 463) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 464) - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 465) +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 466) return unpackUint64(data + at, oSizes); +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 467) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 468) +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 469) // anyTrans specify function +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 470) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 471) void fstStateSetFinalState(FstState* s, bool yes) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 472) assert(s->state == AnyTrans); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 473) if (yes) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 474) s->val |= 0b01000000; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 475) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 476) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 477) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 478) bool fstStateIsFinalState(FstState* s) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 479) assert(s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 480) return (s->val & 0b01000000) == 0b01000000; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 481) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 482) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 483) void fstStateSetStateNtrans(FstState* s, uint8_t n) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 484) assert(s->state == AnyTrans); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 485) if (n <= 0b00111111) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 486) s->val = (s->val & 0b11000000) | n; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 487) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 488) return; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 489) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 490) // state_ntrans +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 491) uint8_t fstStateStateNtrans(FstState* s, bool* null) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 492) assert(s->state == AnyTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 493) *null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 494) uint8_t n = s->val & 0b00111111; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 495) +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 496) if (n == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 497) *null = true; // None +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 498) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 499) return n; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 500) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 501) uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 502) assert(s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 503) uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 504) return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 505) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 506) uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 507) assert(s->state == AnyTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 508) return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 509) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 510) uint64_t fstStateNtransLen(FstState* s) { +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 511) assert(s->state == AnyTrans); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 512) bool null = false; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 513) fstStateStateNtrans(s, &null); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 514) return null == true ? 1 : 0; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 515) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 516) uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 517) bool null = false; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 518) uint8_t n = fstStateStateNtrans(s, &null); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 519) if (null != true) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 520) return n; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 521) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 522) int32_t len; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 523) uint8_t* data = fstSliceData(slice, &len); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 524) n = data[len - 2]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 525) // n = data[slice->end - 1]; // data[data.len() - 2] +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 526) return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 527) // then it is encoded in the state byte +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 528) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 529) Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 530) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 531) if (oSizes == 0 || !fstStateIsFinalState(s)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 532) return 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 533) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 534) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 535) uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 536) - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 537) uint8_t* data = fstSliceData(slice, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 538) return unpackUint64(data + at, (uint8_t)oSizes); +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 539) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 540) uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 541) assert(s->state == AnyTrans); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 542) FstSlice* slice = &node->data; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 543) if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 544) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 545) - fstStateTransIndexSize(s, node->version, node->nTrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 546) int32_t dlen = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 547) uint8_t* data = fstSliceData(slice, &dlen); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 548) uint64_t i = data[at + b]; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 549) // uint64_t i = slice->data[slice->start + at + b]; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 550) if (i >= node->nTrans) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 551) *null = true; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 552) } +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 553) return i; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 554) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 555) uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 556) - node->nTrans; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 557) uint64_t end = start + node->nTrans; +73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 558) FstSlice t = fstSliceCopy(slice, start, end - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 559) int32_t len = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 560) uint8_t* data = fstSliceData(&t, &len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 561) int i = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 562) for (; i < len; i++) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 563) uint8_t v = data[i]; +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 564) if (v == b) { +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 565) fstSliceDestroy(&t); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 566) return node->nTrans - i - 1; // bug +c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 567) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 568) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 569) if (i == len) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 570) *null = true; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 571) } +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 572) fstSliceDestroy(&t); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 573) } +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 574) +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 575) return 0; +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 576) } +169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 577) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 578) // fst node function +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 579) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 580) FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 581) FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 582) if (n == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 583) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 584) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 585) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 586) FstState st = fstStateCreateFrom(slice, addr); +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 587) +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 588) if (st.state == EmptyFinal) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 589) n->data = fstSliceCreate(NULL, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 590) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 591) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 592) n->start = EMPTY_ADDRESS; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 593) n->end = EMPTY_ADDRESS; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 594) n->isFinal = true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 595) n->nTrans = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 596) n->sizes = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 597) n->finalOutput = 0; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 598) } else if (st.state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 599) n->data = fstSliceCopy(slice, 0, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 600) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 601) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 602) n->start = addr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 603) n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 604) n->isFinal = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 605) n->sizes = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 606) n->nTrans = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 607) n->finalOutput = 0; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 608) } else if (st.state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 609) FstSlice data = fstSliceCopy(slice, 0, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 610) PackSizes sz = fstStateSizes(&st, &data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 611) n->data = data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 612) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 613) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 614) n->start = addr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 615) n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 616) n->isFinal = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 617) n->nTrans = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 618) n->sizes = sz; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 619) n->finalOutput = 0; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 620) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 621) FstSlice data = fstSliceCopy(slice, 0, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 622) uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 623) uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 624) n->data = data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 625) n->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 626) n->state = st; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 627) n->start = addr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 628) n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 629) n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 630) n->nTrans = nTrans; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 631) n->sizes = sz; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 632) n->finalOutput = +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 633) fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 634) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 635) return n; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 636) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 637) +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 638) // debug state transition +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 639) static const char* fstNodeState(FstNode* node) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 640) FstState* st = &node->state; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 641) return fstStateStr[st->state]; +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 642) } +55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 643) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 644) void fstNodeDestroy(FstNode* node) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 645) fstSliceDestroy(&node->data); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 646) taosMemoryFree(node); +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 647) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 648) FstTransitions* fstNodeTransitions(FstNode* node) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 649) FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 650) if (NULL == t) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 651) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 652) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 653) FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 654) t->range = range; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 655) t->node = node; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 656) return t; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 657) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 658) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 659) // Returns the transition at index `i`. +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 660) bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 661) bool s = true; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 662) FstState* st = &node->state; +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 663) if (st->state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 664) trn->inp = fstStateInput(st, node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 665) trn->out = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 666) trn->addr = fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 667) } else if (st->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 668) trn->inp = fstStateInput(st, node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 669) trn->out = fstStateOutput(st, node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 670) trn->addr = fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 671) } else if (st->state == AnyTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 672) trn->inp = fstStateInputForAnyTrans(st, node, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 673) trn->out = fstStateOutputForAnyTrans(st, node, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 674) trn->addr = fstStateTransAddrForAnyTrans(st, node, i); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 675) } else { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 676) s = false; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 677) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 678) return s; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 679) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 680) +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 681) // Returns the transition address of the `i`th transition +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 682) bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 683) bool s = true; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 684) FstState* st = &node->state; +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 685) if (st->state == OneTransNext) { +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 686) assert(i == 0); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 687) fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 688) } else if (st->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 689) assert(i == 0); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 690) fstStateTransAddr(st, node); +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 691) } else if (st->state == AnyTrans) { +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 692) fstStateTransAddrForAnyTrans(st, node, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 693) } else if (FST_STATE_EMPTY_FINAL(node)) { +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 694) s = false; +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 695) } else { +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 696) assert(0); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 697) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 698) return s; +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 699) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 700) +dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 701) // Finds the `i`th transition corresponding to the given input byte. +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 702) // If no transition for this byte exists, then `false` is returned. +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 703) bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 704) bool s = true; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 705) FstState* st = &node->state; +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 706) if (st->state == OneTransNext) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 707) if (fstStateInput(st, node) == b) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 708) *res = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 709) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 710) s = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 711) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 712) } else if (st->state == OneTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 713) if (fstStateInput(st, node) == b) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 714) *res = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 715) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 716) s = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 717) } +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 718) } else if (st->state == AnyTrans) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 719) bool null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 720) uint64_t out = fstStateFindInput(st, node, b, &null); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 721) if (null == false) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 722) *res = out; +2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 723) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 724) s = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 725) } +21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 726) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 727) return s; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 728) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 729) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 730) bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 731) size_t sz = taosArrayGetSize(builderNode->trans); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 732) assert(sz < 256); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 733) if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 734) return true; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 735) } else if (sz != 1 || builderNode->isFinal) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 736) fstStateCompileForAnyTrans(w, addr, builderNode); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 737) // AnyTrans->Compile(w, addr, node); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 738) } else { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 739) FstTransition* tran = taosArrayGet(builderNode->trans, 0); +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 740) if (tran->addr == lastAddr && tran->out == 0) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 741) fstStateCompileForOneTransNext(w, addr, tran->inp); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 742) // OneTransNext::compile(w, lastAddr, tran->inp); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 743) return true; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 744) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 745) fstStateCompileForOneTrans(w, addr, tran); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 746) // OneTrans::Compile(w, lastAddr, *tran); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 747) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 748) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 749) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 750) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 751) } +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 752) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 753) bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 754) return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 755) } +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 756) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 757) FstBuilder* fstBuilderCreate(void* w, FstType ty) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 758) FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 759) if (NULL == b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 760) return b; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 761) } +32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 762) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 763) b->wrt = fstCountingWriterCreate(w); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 764) b->unfinished = fstUnFinishedNodesCreate(); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 765) b->registry = fstRegistryCreate(10000, 2); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 766) b->last = fstSliceCreate(NULL, 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 767) b->lastAddr = NONE_ADDRESS; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 768) b->len = 0; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 769) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 770) char buf64[8] = {0}; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 771) void* pBuf64 = buf64; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 772) taosEncodeFixedU64(&pBuf64, VERSION); +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 773) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 774) +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 775) pBuf64 = buf64; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 776) memset(buf64, 0, sizeof(buf64)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 777) taosEncodeFixedU64(&pBuf64, ty); +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 778) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 779) +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 780) return b; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 781) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 782) void fstBuilderDestroy(FstBuilder* b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 783) if (b == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 784) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 785) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 786) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 787) fstCountingWriterDestroy(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 788) fstUnFinishedNodesDestroy(b->unfinished); +260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 789) fstRegistryDestroy(b->registry); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 790) fstSliceDestroy(&b->last); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 791) taosMemoryFree(b); +260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 792) } +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 793) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 794) bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 795) OrderType t = fstBuilderCheckLastKey(b, bs, true); +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 796) if (t == Ordered) { +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 797) // add log info +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 798) fstBuilderInsertOutput(b, bs, in); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 799) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 800) } +ebcb9be39fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-22 12:20:59 +0800 801) indexInfo("fst write key must be ordered"); +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 802) return false; +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 803) } +ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 804) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 805) void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 806) FstSlice* s = &bs; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 807) if (fstSliceIsEmpty(s)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 808) b->len = 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 809) fstUnFinishedNodesSetRootOutput(b->unfinished, in); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 810) return; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 811) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 812) // if (in != 0) { //if let Some(in) = in +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 813) // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 814) //} else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 815) // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 816) // out = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 817) //} +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 818) Output out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 819) uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 820) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 821) if (prefixLen == FST_SLICE_LEN(s)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 822) assert(out == 0); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 823) return; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 824) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 825) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 826) b->len += 1; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 827) fstBuilderCompileFrom(b, prefixLen); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 828) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 829) FstSlice sub = fstSliceCopy(s, prefixLen, s->end); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 830) fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 831) fstSliceDestroy(&sub); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 832) return; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 833) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 834) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 835) OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 836) FstSlice* input = &bs; +716c0045f8a source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 17:29:59 +0800 837) if (fstSliceIsEmpty(&b->last)) { +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 838) fstSliceDestroy(&b->last); +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 839) // deep copy or not +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 840) b->last = fstSliceDeepCopy(&bs, input->start, input->end); +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 841) } else { +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 842) int comp = fstSliceCompare(&b->last, &bs); +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 843) if (comp == 0 && ckDup) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 844) return DuplicateKey; +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 845) } else if (comp == 1) { +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 846) return OutOfOrdered; +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 847) } +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 848) // deep copy or not +d8d6c04fd50 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 16:18:35 +0800 849) fstSliceDestroy(&b->last); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 850) b->last = fstSliceDeepCopy(&bs, input->start, input->end); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 851) } +e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 852) return Ordered; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 853) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 854) void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 855) CompiledAddr addr = NONE_ADDRESS; +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 856) while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 857) FstBuilderNode* bn = NULL; +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 858) if (addr == NONE_ADDRESS) { +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 859) bn = fstUnFinishedNodesPopEmpty(b->unfinished); +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 860) } else { +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 861) bn = fstUnFinishedNodesPopFreeze(b->unfinished, addr); +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 862) } +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 863) addr = fstBuilderCompile(b, bn); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 864) +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 865) fstBuilderNodeDestroy(bn); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 866) assert(addr != NONE_ADDRESS); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 867) // fstBuilderNodeDestroy(n); +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 868) } +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 869) fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 870) return; +5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 871) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 872) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 873) CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 874) if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 875) return EMPTY_ADDRESS; +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 876) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 877) FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 878) if (entry->state == FOUND) { +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 879) CompiledAddr ret = entry->addr; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 880) fstRegistryEntryDestroy(entry); +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 881) return ret; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 882) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 883) CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt)); +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 884) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 885) fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 886) b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 887) if (entry->state == NOTFOUND) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 888) FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 889) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 890) fstRegistryEntryDestroy(entry); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 891) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 892) return b->lastAddr; +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 893) } +0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 894) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 895) void* fstBuilderInsertInner(FstBuilder* b) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 896) fstBuilderCompileFrom(b, 0); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 897) FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 898) CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 899) fstBuilderNodeDestroy(rootNode); +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 900) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 901) char buf64[8] = {0}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 902) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 903) void* pBuf64 = buf64; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 904) taosEncodeFixedU64(&pBuf64, b->len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 905) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 906) +d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 907) pBuf64 = buf64; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 908) taosEncodeFixedU64(&pBuf64, rootAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 909) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 910) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 911) char buf32[4] = {0}; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 912) void* pBuf32 = buf32; +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 913) uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 914) taosEncodeFixedU32(&pBuf32, sum); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 915) fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 916) +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 917) fstCountingWriterFlush(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 918) // fstCountingWriterDestroy(b->wrt); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 919) // b->wrt = NULL; +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 920) return b->wrt; +e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 921) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 922) void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 923) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 924) FstSlice fstNodeAsSlice(FstNode* node) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 925) FstSlice* slice = &node->data; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 926) FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 927) return s; +60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 928) } +980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 929) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 930) FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 931) FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 932) if (trn == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 933) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 934) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 935) +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 936) trn->inp = inp; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 937) trn->out = out; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 938) return trn; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 939) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 940) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 941) void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } +22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 942) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 943) void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 944) FstLastTransition* trn = unNode->last; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 945) if (trn == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 946) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 947) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 948) FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 949) taosArrayPush(unNode->node->trans, &t); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 950) fstLastTransitionDestroy(trn); +7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 951) unNode->last = NULL; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 952) return; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 953) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 954) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 955) void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 956) if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 957) unNode->node->finalOutput += out; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 958) } +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 959) size_t sz = taosArrayGetSize(unNode->node->trans); +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 960) for (size_t i = 0; i < sz; i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 961) FstTransition* trn = taosArrayGet(unNode->node->trans, i); +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 962) trn->out += out; +06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 963) } +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 964) if (unNode->last) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 965) unNode->last->out += out; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 966) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 967) return; +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 968) } +0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 969) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 970) Fst* fstCreate(FstSlice* slice) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 971) int32_t slen; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 972) char* buf = fstSliceData(slice, &slen); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 973) if (slen < 36) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 974) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 975) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 976) uint64_t len = slen; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 977) uint64_t skip = 0; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 978) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 979) uint64_t version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 980) taosDecodeFixedU64(buf, &version); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 981) skip += sizeof(version); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 982) if (version == 0 || version > VERSION) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 983) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 984) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 985) +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 986) uint64_t type; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 987) taosDecodeFixedU64(buf + skip, &type); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 988) skip += sizeof(type); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 989) +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 990) uint32_t checkSum = 0; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 991) len -= sizeof(checkSum); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 992) taosDecodeFixedU32(buf + len, &checkSum); +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 993) if (taosCheckChecksum(buf, len, checkSum)) { +f9d83c044dd source/libs/index/src/index_fst.c (yihaoDeng 2022-01-10 23:42:58 +0800 994) indexError("index file is corrupted"); +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 995) // verify fst +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 996) return NULL; +15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 997) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 998) CompiledAddr rootAddr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 999) len -= sizeof(rootAddr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1000) taosDecodeFixedU64(buf + len, &rootAddr); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1001) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1002) uint64_t fstLen; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1003) len -= sizeof(fstLen); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1004) taosDecodeFixedU64(buf + len, &fstLen); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1005) // TODO(validate root addr) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1006) Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1007) if (fst == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1008) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1009) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1010) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1011) fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1012) if (NULL == fst->meta) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1013) goto FST_CREAT_FAILED; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1014) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1015) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1016) fst->meta->version = version; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1017) fst->meta->rootAddr = rootAddr; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1018) fst->meta->ty = type; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1019) fst->meta->len = fstLen; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1020) fst->meta->checkSum = checkSum; +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1021) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1022) FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1023) *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1024) fst->data = s; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1025) +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1026) taosThreadMutexInit(&fst->mtx, NULL); +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1027) return fst; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1028) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1029) FST_CREAT_FAILED: +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1030) taosMemoryFree(fst->meta); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1031) taosMemoryFree(fst); +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1032) +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1033) return NULL; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1034) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1035) void fstDestroy(Fst* fst) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1036) if (fst) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1037) taosMemoryFree(fst->meta); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1038) fstSliceDestroy(fst->data); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1039) taosMemoryFree(fst->data); +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1040) taosThreadMutexDestroy(&fst->mtx); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1041) } +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1042) taosMemoryFree(fst); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1043) } +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1044) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1045) bool fstGet(Fst* fst, FstSlice* b, Output* out) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1046) // dec lock range +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1047) // taosThreadMutexLock(&fst->mtx); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1048) FstNode* root = fstGetRoot(fst); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1049) Output tOut = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1050) int32_t len; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1051) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1052) uint8_t* data = fstSliceData(b, &len); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1053) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1054) SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1055) taosArrayPush(nodes, &root); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1056) for (uint32_t i = 0; i < len; i++) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1057) uint8_t inp = data[i]; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1058) Output res = 0; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1059) if (false == fstNodeFindInput(root, inp, &res)) { +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1060) // taosThreadMutexUnlock(&fst->mtx); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1061) return false; +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1062) } +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1063) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1064) FstTransition trn; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1065) fstNodeGetTransitionAt(root, res, &trn); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1066) tOut += trn.out; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1067) root = fstGetNode(fst, trn.addr); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1068) taosArrayPush(nodes, &root); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1069) } +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1070) if (!FST_NODE_IS_FINAL(root)) { +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1071) // taosThreadMutexUnlock(&fst->mtx); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1072) return false; +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1073) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1074) tOut = tOut + FST_NODE_FINAL_OUTPUT(root); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1075) } +f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 1076) +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1077) for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1078) FstNode** node = (FstNode**)taosArrayGet(nodes, i); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1079) fstNodeDestroy(*node); +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1080) } +29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1081) taosArrayDestroy(nodes); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1082) // fst->root = NULL; +79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1083) // taosThreadMutexUnlock(&fst->mtx); +2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1084) *out = tOut; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1085) return true; +f07045c1d8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 17:28:58 +0800 1086) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1087) FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1088) // refactor later +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1089) return fstStreamBuilderCreate(fst, ctx); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1090) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1091) StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1092) if (sb == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1093) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1094) } +26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1095) return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); +26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1096) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1097) FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1098) // refactor later +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1099) return fstStreamBuilderCreate(fst, ctx); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1100) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1101) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1102) FstNode* fstGetRoot(Fst* fst) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1103) CompiledAddr rAddr = fstGetRootAddr(fst); +2eb0053e5ad source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 22:19:56 +0800 1104) return fstGetNode(fst, rAddr); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1105) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1106) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1107) FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1108) // refactor later +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1109) return fstNodeCreate(fst->meta->version, addr, fst->data); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1110) } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1111) FstType fstGetType(Fst* fst) { return fst->meta->ty; } +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1112) CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; } +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1113) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1114) Output fstEmptyFinalOutput(Fst* fst, bool* null) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1115) Output res = 0; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1116) FstNode* node = fstGetRoot(fst); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1117) if (FST_NODE_IS_FINAL(node)) { +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1118) *null = false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1119) res = FST_NODE_FINAL_OUTPUT(node); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1120) } else { +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1121) *null = true; +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1122) } +40fc4b6cf38 source/libs/index/src/index_fst.c (yihaoDeng 2022-01-02 17:50:40 +0800 1123) fstNodeDestroy(node); +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1124) return res; +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1125) } +500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1126) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1127) bool fstVerify(Fst* fst) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1128) uint32_t len, checkSum = fst->meta->checkSum; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1129) uint8_t* data = fstSliceData(fst->data, &len); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1130) TSCKSUM initSum = 0; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1131) if (!taosCheckChecksumWhole(data, len)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1132) return false; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1133) } +a09dcbdd85f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:59:51 +0800 1134) return true; +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1135) } +c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1136) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1137) // data bound function +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1138) FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1139) FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1140) if (b == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1141) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1142) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1143) +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1144) if (data != NULL) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1145) b->data = fstSliceCopy(data, data->start, data->end); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1146) } else { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1147) b->data = fstSliceCreate(NULL, 0); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1148) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1149) b->type = type; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1150) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1151) return b; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1152) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1153) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1154) bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1155) int comp = fstSliceCompare(slice, &bound->data); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1156) if (bound->type == Included) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1157) return comp > 0 ? true : false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1158) } else if (bound->type == Excluded) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1159) return comp >= 0 ? true : false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1160) } else { +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1161) return false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1162) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1163) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1164) bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1165) if (bound->type == Unbounded) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1166) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1167) } else { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1168) return fstSliceIsEmpty(&bound->data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1169) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1170) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1171) +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1172) bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1173) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1174) void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1175) +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1176) StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1177) FstBoundWithData* max) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1178) StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1179) if (sws == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1180) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1181) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1182) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1183) sws->fst = fst; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1184) sws->aut = automation; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1185) sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1186) +0c5f2d1da95 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 21:41:41 +0800 1187) sws->emptyOutput.null = true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1188) sws->emptyOutput.out = 0; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1189) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1190) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1191) sws->endAt = max; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1192) streamWithStateSeekMin(sws, min); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1193) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1194) return sws; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1195) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1196) void streamWithStateDestroy(StreamWithState* sws) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1197) if (sws == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1198) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1199) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1200) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1201) taosArrayDestroy(sws->inp); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1202) taosArrayDestroyEx(sws->stack, streamStateDestroy); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1203) +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1204) taosMemoryFree(sws); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1205) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1206) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1207) bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1208) AutomationCtx* aut = sws->aut; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1209) if (fstBoundWithDataIsEmpty(min)) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1210) if (fstBoundWithDataIsIncluded(min)) { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1211) sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1212) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1213) StreamState s = {.node = fstGetRoot(sws->fst), +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1214) .trans = 0, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1215) .out = {.null = false, .out = 0}, +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1216) .autState = automFuncs[aut->type].start(aut)}; // auto.start callback +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1217) taosArrayPush(sws->stack, &s); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1218) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1219) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1220) FstSlice* key = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1221) bool inclusize = false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1222) +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1223) if (min->type == Included) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1224) key = &min->data; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1225) inclusize = true; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1226) } else if (min->type == Excluded) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1227) key = &min->data; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1228) } else { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1229) return false; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1230) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1231) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1232) FstNode* node = fstGetRoot(sws->fst); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1233) Output out = 0; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1234) // void* autState = sws->aut->start(); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1235) void* autState = automFuncs[aut->type].start(aut); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1236) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1237) int32_t len; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1238) uint8_t* data = fstSliceData(key, &len); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1239) for (uint32_t i = 0; i < len; i++) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1240) uint8_t b = data[i]; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1241) uint64_t res = 0; +2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1242) bool find = fstNodeFindInput(node, b, &res); +2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1243) if (find == true) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1244) FstTransition trn; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1245) fstNodeGetTransitionAt(node, res, &trn); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1246) void* preState = autState; +cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1247) // autState = sws->aut->accept(preState, b); +cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1248) autState = automFuncs[aut->type].accept(aut, preState, b); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1249) taosArrayPush(sws->inp, &b); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1250) StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1251) taosArrayPush(sws->stack, &s); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1252) out += trn.out; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1253) node = fstGetNode(sws->fst, trn.addr); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1254) } else { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1255) // This is a little tricky. We're in this case if the +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1256) // given bound is not a prefix of any key in the FST. +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1257) // Since this is a minimum bound, we need to find the +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1258) // first transition in this node that proceeds the current +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1259) // input byte. +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1260) FstTransitions* trans = fstNodeTransitions(node); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1261) uint64_t i = 0; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1262) for (i = trans->range.start; i < trans->range.end; i++) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1263) FstTransition trn; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1264) if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1265) break; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1266) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1267) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1268) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1269) StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1270) taosArrayPush(sws->stack, &s); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1271) return true; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1272) } +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1273) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1274) uint32_t sz = taosArrayGetSize(sws->stack); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1275) if (sz != 0) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1276) StreamState* s = taosArrayGet(sws->stack, sz - 1); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1277) if (inclusize) { +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1278) s->trans -= 1; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1279) taosArrayPop(sws->inp); +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1280) } else { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1281) FstNode* n = s->node; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1282) uint64_t trans = s->trans; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1283) FstTransition trn; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1284) fstNodeGetTransitionAt(n, trans - 1, &trn); +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1285) StreamState s = { +236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1286) .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; +23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1287) taosArrayPush(sws->stack, &s); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1288) return true; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1289) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1290) return false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1291) } +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1292) +23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1293) return false; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1294) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1295) StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1296) AutomationCtx* aut = sws->aut; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1297) FstOutput output = sws->emptyOutput; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1298) if (output.null == false) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1299) FstSlice emptySlice = fstSliceCreate(NULL, 0); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1300) if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1301) taosArrayDestroyEx(sws->stack, streamStateDestroy); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1302) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1303) return NULL; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1304) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1305) void* start = automFuncs[aut->type].start(aut); +cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1306) if (automFuncs[aut->type].isMatch(aut, start)) { +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1307) FstSlice s = fstSliceCreate(NULL, 0); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1308) return swsResultCreate(&s, output, callback == NULL ? NULL : callback(start)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1309) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1310) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1311) SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1312) while (taosArrayGetSize(sws->stack) > 0) { +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1313) StreamState* p = (StreamState*)taosArrayPop(sws->stack); +f0110af30ec source/libs/index/src/index_fst.c (yihaoDeng 2021-12-16 22:04:47 +0800 1314) if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1315) if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1316) taosArrayPop(sws->inp); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1317) } +6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1318) streamStateDestroy(p); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1319) continue; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1320) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1321) FstTransition trn; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1322) fstNodeGetTransitionAt(p->node, p->trans, &trn); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1323) +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1324) Output out = p->out.out + trn.out; +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1325) void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1326) void* tState = (callback == NULL) ? NULL : callback(nextState); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1327) bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); +f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1328) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1329) FstNode* nextNode = fstGetNode(sws->fst, trn.addr); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1330) taosArrayPush(nodes, &nextNode); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1331) taosArrayPush(sws->inp, &(trn.inp)); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1332) +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1333) if (FST_NODE_IS_FINAL(nextNode)) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1334) // void *eofState = sws->aut->acceptEof(nextState); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1335) void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1336) if (eofState != NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1337) isMatch = automFuncs[aut->type].isMatch(aut, eofState); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1338) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1339) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1340) StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1341) taosArrayPush(sws->stack, &s1); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1342) +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1343) StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1344) taosArrayPush(sws->stack, &s2); +6f3c49ee73d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-14 16:17:38 +0800 1345) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1346) size_t isz = taosArrayGetSize(sws->inp); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1347) uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1348) for (uint32_t i = 0; i < isz; i++) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1349) buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1350) } +00000000000 source/libs/index/src/indexFst.c (Not Committed Yet 2022-04-02 15:19:56 +0800 1351) FstSlice slice = fstSliceCreate(buf, isz); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1352) if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1353) taosArrayDestroyEx(sws->stack, streamStateDestroy); +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1354) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1355) taosMemoryFreeClear(buf); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1356) fstSliceDestroy(&slice); +06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1357) taosArrayDestroy(nodes); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1358) return NULL; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1359) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1360) if (FST_NODE_IS_FINAL(nextNode) && isMatch) { +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1361) FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1362) StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1363) taosMemoryFreeClear(buf); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1364) fstSliceDestroy(&slice); +06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1365) taosArrayDestroy(nodes); +6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1366) nodes = NULL; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1367) return result; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1368) } +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1369) taosMemoryFreeClear(buf); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1370) fstSliceDestroy(&slice); +6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1371) }; +06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1372) taosArrayDestroy(nodes); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1373) return NULL; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1374) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1375) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1376) StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1377) StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1378) if (result == NULL) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1379) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1380) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1381) +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1382) result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1383) result->out = fOut; +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1384) result->state = state; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1385) return result; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1386) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1387) void swsResultDestroy(StreamWithStateResult* result) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1388) if (NULL == result) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1389) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1390) } +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1391) +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1392) fstSliceDestroy(&result->data); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1393) startWithStateValueDestroy(result->state); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1394) taosMemoryFree(result); +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1395) } +e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1396) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1397) void streamStateDestroy(void* s) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1398) if (NULL == s) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1399) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1400) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1401) StreamState* ss = (StreamState*)s; +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1402) fstNodeDestroy(ss->node); +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1403) } +fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1404) +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1405) FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1406) FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1407) if (NULL == b) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1408) return NULL; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1409) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1410) +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1411) b->fst = fst; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1412) b->aut = aut; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1413) b->min = fstBoundStateCreate(Unbounded, NULL); +5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1414) b->max = fstBoundStateCreate(Unbounded, NULL); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1415) return b; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1416) } +10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1417) void fstStreamBuilderDestroy(FstStreamBuilder* b) { +7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1418) fstSliceDestroy(&b->min->data); +7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1419) fstSliceDestroy(&b->max->data); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1420) taosMemoryFreeClear(b->min); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1421) taosMemoryFreeClear(b->max); +222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1422) taosMemoryFree(b); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1423) } +30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1424) void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1425) if (b == NULL) { +30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1426) return; +9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1427) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1428) if (type == GE) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1429) b->min->type = Included; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1430) fstSliceDestroy(&(b->min->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1431) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1432) } else if (type == GT) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1433) b->min->type = Excluded; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1434) fstSliceDestroy(&(b->min->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1435) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1436) } else if (type == LE) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1437) b->max->type = Included; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1438) fstSliceDestroy(&(b->max->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1439) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1440) } else if (type == LT) { +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1441) b->max->type = Excluded; +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1442) fstSliceDestroy(&(b->max->data)); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1443) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1444) } +d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1445) } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index eff53108cd..c45f655746 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -510,6 +510,68 @@ void checkFstCheckIteratorRange2() { } delete m; } +void checkFstCheckIteratorRange3() { + FstWriter* fw = new FstWriter; + int64_t s = taosGetTimestampUs(); + int count = 2; + // Performance_fstWriteRecords(fw); + int64_t e = taosGetTimestampUs(); + + std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl; + + fw->Put("ab", 1); + fw->Put("b", 2); + fw->Put("cdd", 3); + fw->Put("cde", 3); + fw->Put("ddd", 4); + fw->Put("ed", 5); + delete fw; + + FstReadMemory* m = new FstReadMemory(1024 * 64); + if (m->init() == false) { + std::cout << "init readMemory failed" << std::endl; + delete m; + return; + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "b", GE, "", (RangeType)10, result); + assert(result.size() == 5); + automCtxDestroy(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "", (RangeType)20, "ab", LE, result); + assert(result.size() == 1); + automCtxDestroy(ctx); + // taosMemoryFree(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "", (RangeType)30, "ab", LT, result); + assert(result.size() == 0); + automCtxDestroy(ctx); + } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "ed", GT, "ed", (RangeType)40, result); + assert(result.size() == 0); + automCtxDestroy(ctx); + } + delete m; +} void fst_get(Fst* fst) { for (int i = 0; i < 10000; i++) { @@ -573,11 +635,12 @@ int main(int argc, char* argv[]) { // path suid colName ver // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); //} - checkFstCheckIterator1(); - checkFstCheckIterator2(); - checkFstCheckIteratorPrefix(); - checkFstCheckIteratorRange1(); - checkFstCheckIteratorRange2(); + // checkFstCheckIterator1(); + // checkFstCheckIterator2(); + // checkFstCheckIteratorPrefix(); + // checkFstCheckIteratorRange1(); + // checkFstCheckIteratorRange2(); + checkFstCheckIteratorRange3(); // checkFstLongTerm(); // checkFstPrefixSearch(); From 3a3fef91f0828ecbb54fbfbf83b0233a53dfc381 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 15:25:44 +0800 Subject: [PATCH 13/42] handle except --- source/libs/index/src/log | 1445 ------------------------------------- 1 file changed, 1445 deletions(-) delete mode 100644 source/libs/index/src/log diff --git a/source/libs/index/src/log b/source/libs/index/src/log deleted file mode 100644 index 0b6cb8ddc2..0000000000 --- a/source/libs/index/src/log +++ /dev/null @@ -1,1445 +0,0 @@ -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 1) /* -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 2) * Copyright (c) 2019 TAOS Data, Inc. -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 3) * -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 4) * This program is free software: you can use, redistribute, and/or modify -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 5) * it under the terms of the GNU Affero General Public License, version 3 -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 6) * or later ("AGPL"), as published by the Free Software Foundation. -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 7) * -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 8) * This program is distributed in the hope that it will be useful, but WITHOUT -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 9) * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 10) * FITNESS FOR A PARTICULAR PURPOSE. -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 11) * -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 12) * You should have received a copy of the GNU Affero General Public License -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 13) * along with this program. If not, see . -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 14) */ -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 15) -5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 16) #include "indexFst.h" -5cf0c4a61c6 source/libs/index/src/indexFst.c (yihaoDeng 2022-03-29 23:11:57 +0800 17) #include "indexFstAutomation.h" -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 18) #include "indexInt.h" -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 19) #include "tchecksum.h" -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 20) #include "tcoding.h" -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 21) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 22) static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 23) CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 24) fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 25) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 26) static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { -20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 27) uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 28) fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 29) return nBytes; -20203e47eb5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:57:50 +0800 30) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 31) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 32) FstUnFinishedNodes* fstUnFinishedNodesCreate() { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 33) FstUnFinishedNodes* nodes = taosMemoryMalloc(sizeof(FstUnFinishedNodes)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 34) if (nodes == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 35) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 36) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 37) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 38) nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 39) fstUnFinishedNodesPushEmpty(nodes, false); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 40) return nodes; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 41) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 42) static void unFinishedNodeDestroyElem(void* elem) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 43) FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 44) fstBuilderNodeDestroy(b->node); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 45) taosMemoryFree(b->last); -7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 46) b->last = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 47) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 48) void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 49) if (nodes == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 50) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 51) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 52) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 53) taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 54) taosMemoryFree(nodes); -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 55) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 56) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 57) void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 58) FstBuilderNode* node = taosMemoryMalloc(sizeof(FstBuilderNode)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 59) node->isFinal = isFinal; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 60) node->finalOutput = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 61) node->trans = taosArrayInit(16, sizeof(FstTransition)); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 62) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 63) FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 64) taosArrayPush(nodes->stack, &un); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 65) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 66) FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 67) assert(taosArrayGetSize(nodes->stack) == 1); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 68) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 69) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 70) assert(un->last == NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 71) return un->node; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 72) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 73) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 74) FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 75) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 76) fstBuilderNodeUnfinishedLastCompiled(un, addr); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 77) // taosMemoryFree(un->last); // TODO add func FstLastTransitionFree() -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 78) // un->last = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 79) return un->node; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 80) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 81) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 82) FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 83) FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 84) assert(un->last == NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 85) return un->node; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 86) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 87) void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 88) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 89) un->node->isFinal = true; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 90) un->node->finalOutput = out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 91) // un->node->trans = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 92) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 93) void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 94) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 95) fstBuilderNodeUnfinishedLastCompiled(un, addr); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 96) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 97) void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 98) FstSlice* s = &bs; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 99) if (fstSliceIsEmpty(s)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 100) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 101) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 102) size_t sz = taosArrayGetSize(nodes->stack) - 1; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 103) FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 104) assert(un->last == NULL); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 105) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 106) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 107) // trn->inp = s->data[s->start]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 108) // trn->out = out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 109) int32_t len = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 110) uint8_t* data = fstSliceData(s, &len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 111) un->last = fstLastTransitionCreate(data[0], out); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 112) -d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 113) for (uint64_t i = 1; i < len; i++) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 114) FstBuilderNode* n = taosMemoryMalloc(sizeof(FstBuilderNode)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 115) n->isFinal = false; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 116) n->finalOutput = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 117) n->trans = taosArrayInit(16, sizeof(FstTransition)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 118) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 119) // FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 120) // trn->inp = s->data[i]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 121) // trn->out = out; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 122) FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 123) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 124) FstBuilderNodeUnfinished un = {.node = n, .last = trn}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 125) taosArrayPush(nodes->stack, &un); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 126) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 127) fstUnFinishedNodesPushEmpty(nodes, true); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 128) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 129) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 130) uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 131) FstSlice* s = &bs; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 132) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 133) size_t ssz = taosArrayGetSize(node->stack); // stack size -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 134) uint64_t count = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 135) int32_t lsz; // data len -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 136) uint8_t* data = fstSliceData(s, &lsz); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 137) for (size_t i = 0; i < ssz && i < lsz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 138) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 139) if (un->last->inp == data[i]) { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 140) count++; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 141) } else { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 142) break; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 143) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 144) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 145) return count; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 146) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 147) uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 148) FstSlice* s = &bs; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 149) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 150) size_t lsz = (size_t)(s->end - s->start + 1); // data len -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 151) size_t ssz = taosArrayGetSize(node->stack); // stack size -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 152) *out = in; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 153) uint64_t i = 0; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 154) for (i = 0; i < lsz && i < ssz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 155) FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 156) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 157) FstLastTransition* t = un->last; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 158) uint64_t addPrefix = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 159) uint8_t* data = fstSliceData(s, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 160) if (t && t->inp == data[i]) { -92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 161) uint64_t commPrefix = TMIN(t->out, *out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 162) uint64_t tAddPrefix = t->out - commPrefix; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 163) (*out) = (*out) - commPrefix; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 164) t->out = commPrefix; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 165) addPrefix = tAddPrefix; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 166) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 167) break; -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 168) } -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 169) if (addPrefix != 0) { -73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 170) if (i + 1 < ssz) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 171) FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 172) fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); -73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 173) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 174) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 175) } -16f089c39a7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 15:24:55 +0800 176) return i; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 177) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 178) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 179) FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { -93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 180) FstState fs = {.state = EmptyFinal, .val = 0}; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 181) if (addr == EMPTY_ADDRESS) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 182) return fs; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 183) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 184) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 185) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 186) uint8_t v = data[addr]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 187) uint8_t t = (v & 0b11000000) >> 6; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 188) if (t == 0b11) { -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 189) fs.state = OneTransNext; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 190) } else if (t == 0b10) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 191) fs.state = OneTrans; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 192) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 193) fs.state = AnyTrans; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 194) } -93c102e2940 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 14:15:23 +0800 195) fs.val = v; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 196) return fs; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 197) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 198) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 199) static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 200) {.state = OneTrans, .val = 0b10000000}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 201) {.state = AnyTrans, .val = 0b00000000}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 202) {.state = EmptyFinal, .val = 0b00000000}}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 203) // debug -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 204) static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 205) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 206) FstState fstStateCreate(State state) { -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 207) uint8_t idx = (uint8_t)state; -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 208) return fstStateDict[idx]; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 209) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 210) // compile -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 211) void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 212) FstState s = fstStateCreate(OneTransNext); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 213) fstStateSetCommInput(&s, inp); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 214) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 215) bool null = false; -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 216) uint8_t v = fstStateCommInput(&s, &null); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 217) if (null) { -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 218) // w->write_all(&[inp]) -7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 219) fstCountingWriterWrite(w, &inp, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 220) } -7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 221) fstCountingWriterWrite(w, &(s.val), 1); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 222) // w->write_all(&[s.val]) -7b62d02f95e source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:17:37 +0800 223) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 224) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 225) void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 226) Output out = trn->out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 227) uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 228) uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 229) PackSizes packSizes = 0; -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 230) -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 231) FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 232) FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 233) fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 234) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 235) FstState st = fstStateCreate(OneTrans); -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 236) -d39f80185f7 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 20:42:19 +0800 237) fstStateSetCommInput(&st, trn->inp); -22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 238) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 239) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 240) uint8_t inp = fstStateCommInput(&st, &null); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 241) if (null == true) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 242) fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 243) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 244) fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 245) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 246) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 247) void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 248) size_t sz = taosArrayGetSize(node->trans); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 249) assert(sz <= 256); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 250) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 251) uint8_t tSize = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 252) uint8_t oSize = packSize(node->finalOutput); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 253) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 254) // finalOutput.is_zero() -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 255) bool anyOuts = (node->finalOutput != 0); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 256) for (size_t i = 0; i < sz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 257) FstTransition* t = taosArrayGet(node->trans, i); -92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 258) tSize = TMAX(tSize, packDeltaSize(addr, t->addr)); -92bef71ec7e source/libs/index/src/index_fst.c (yihaoDeng 2022-01-24 12:53:17 +0800 259) oSize = TMAX(oSize, packSize(t->out)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 260) anyOuts = anyOuts || (t->out != 0); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 261) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 262) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 263) PackSizes packSizes = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 264) if (anyOuts) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 265) FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 266) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 267) FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 268) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 269) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 270) FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 271) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 272) FstState st = fstStateCreate(AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 273) fstStateSetFinalState(&st, node->isFinal); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 274) fstStateSetStateNtrans(&st, (uint8_t)sz); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 275) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 276) if (anyOuts) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 277) if (FST_BUILDER_NODE_IS_FINAL(node)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 278) fstCountingWriterPackUintIn(w, node->finalOutput, oSize); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 279) } -d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 280) for (int32_t i = sz - 1; i >= 0; i--) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 281) FstTransition* t = taosArrayGet(node->trans, i); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 282) fstCountingWriterPackUintIn(w, t->out, oSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 283) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 284) } -d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 285) for (int32_t i = sz - 1; i >= 0; i--) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 286) FstTransition* t = taosArrayGet(node->trans, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 287) fstPackDeltaIn(w, addr, t->addr, tSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 288) } -d266c6f29f3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 21:58:33 +0800 289) for (int32_t i = sz - 1; i >= 0; i--) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 290) FstTransition* t = taosArrayGet(node->trans, i); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 291) fstCountingWriterWrite(w, (char*)&t->inp, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 292) // fstPackDeltaIn(w, addr, t->addr, tSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 293) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 294) if (sz > TRANS_INDEX_THRESHOLD) { -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 295) // A value of 255 indicates that no transition exists for the byte -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 296) // at that index. (Except when there are 256 transitions.) Namely, -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 297) // any value greater than or equal to the number of transitions in -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 298) // this node indicates an absent transition. -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 299) uint8_t* index = (uint8_t*)taosMemoryMalloc(sizeof(uint8_t) * 256); -984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 300) memset(index, 255, sizeof(uint8_t) * 256); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 301) /// for (uint8_t i = 0; i < 256; i++) { -984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 302) // index[i] = 255; -984f3023537 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 20:08:40 +0800 303) ///} -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 304) for (size_t i = 0; i < sz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 305) FstTransition* t = taosArrayGet(node->trans, i); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 306) index[t->inp] = i; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 307) // fstPackDeltaIn(w, addr, t->addr, tSize); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 308) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 309) fstCountingWriterWrite(w, (char*)index, 256); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 310) taosMemoryFree(index); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 311) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 312) fstCountingWriterWrite(w, (char*)&packSizes, 1); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 313) bool null = false; -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 314) fstStateStateNtrans(&st, &null); -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 315) if (null == true) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 316) // 256 can't be represented in a u8, so we abuse the fact that -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 317) // the # of transitions can never be 1 here, since 1 is always -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 318) // encoded in the state byte. -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 319) uint8_t v = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 320) if (sz == 256) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 321) fstCountingWriterWrite(w, (char*)&v, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 322) } else { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 323) fstCountingWriterWrite(w, (char*)&sz, 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 324) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 325) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 326) fstCountingWriterWrite(w, (char*)(&(st.val)), 1); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 327) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 328) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 329) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 330) // set_comm_input -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 331) void fstStateSetCommInput(FstState* s, uint8_t inp) { -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 332) assert(s->state == OneTransNext || s->state == OneTrans); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 333) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 334) uint8_t val; -9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 335) COMMON_INDEX(inp, 0b111111, val); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 336) s->val = (s->val & fstStateDict[s->state].val) | val; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 337) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 338) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 339) // comm_input -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 340) uint8_t fstStateCommInput(FstState* s, bool* null) { -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 341) assert(s->state == OneTransNext || s->state == OneTrans); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 342) uint8_t v = s->val & 0b00111111; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 343) if (v == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 344) *null = true; -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 345) return v; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 346) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 347) // v = 0 indicate that common_input is None -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 348) return v == 0 ? 0 : COMMON_INPUT(v); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 349) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 350) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 351) // input_len -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 352) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 353) uint64_t fstStateInputLen(FstState* s) { -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 354) assert(s->state == OneTransNext || s->state == OneTrans); -d674fcc8bab source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 20:03:20 +0800 355) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 356) fstStateCommInput(s, &null); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 357) return null ? 1 : 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 358) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 359) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 360) // end_addr -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 361) uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { -4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 362) assert(s->state == OneTransNext); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 363) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 364) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 365) uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { -4e9fba6dd31 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 14:37:32 +0800 366) assert(s->state == OneTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 367) return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 368) - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 369) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 370) uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 371) uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 372) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 373) uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 374) return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 375) - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 376) - finalOsize; // final output -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 377) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 378) // input -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 379) uint8_t fstStateInput(FstState* s, FstNode* node) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 380) assert(s->state == OneTransNext || s->state == OneTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 381) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 382) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 383) uint8_t inp = fstStateCommInput(s, &null); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 384) uint8_t* data = fstSliceData(slice, NULL); -9920b43be59 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-31 18:06:13 +0800 385) return null == false ? inp : data[node->start - 1]; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 386) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 387) uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 388) assert(s->state == AnyTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 389) FstSlice* slice = &node->data; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 390) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 391) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 392) - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 393) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 394) uint8_t* data = fstSliceData(slice, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 395) return data[at]; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 396) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 397) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 398) // trans_addr -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 399) CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 400) assert(s->state == OneTransNext || s->state == OneTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 401) FstSlice* slice = &node->data; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 402) if (s->state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 403) return (CompiledAddr)(node->end) - 1; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 404) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 405) PackSizes sizes = node->sizes; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 406) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 407) uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 408) - tSizes; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 409) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 410) // refactor error logic -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 411) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 412) return unpackDelta(data + i, tSizes, node->end); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 413) } -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 414) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 415) CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 416) assert(s->state == AnyTrans); -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 417) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 418) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 419) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 420) uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 421) node->nTrans - (i * tSizes) - tSizes; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 422) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 423) return unpackDelta(data + at, tSizes, node->end); -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 424) } -0aa47daf89c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 20:01:19 +0800 425) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 426) // sizes -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 427) PackSizes fstStateSizes(FstState* s, FstSlice* slice) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 428) assert(s->state == OneTrans || s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 429) uint64_t i; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 430) if (s->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 431) i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 432) } else { -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 433) i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 434) } -169d9e17ed8 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 14:05:25 +0800 435) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 436) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 437) return (PackSizes)(*(data + i)); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 438) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 439) // Output -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 440) Output fstStateOutput(FstState* s, FstNode* node) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 441) assert(s->state == OneTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 442) -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 443) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 444) if (oSizes == 0) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 445) return 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 446) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 447) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 448) uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 449) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 450) uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 451) uint8_t* data = fstSliceData(slice, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 452) return unpackUint64(data + i, oSizes); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 453) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 454) Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 455) assert(s->state == AnyTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 456) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 457) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 458) if (oSizes == 0) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 459) return 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 460) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 461) FstSlice* slice = &node->data; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 462) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 463) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 464) - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 465) -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 466) return unpackUint64(data + at, oSizes); -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 467) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 468) -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 469) // anyTrans specify function -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 470) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 471) void fstStateSetFinalState(FstState* s, bool yes) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 472) assert(s->state == AnyTrans); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 473) if (yes) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 474) s->val |= 0b01000000; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 475) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 476) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 477) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 478) bool fstStateIsFinalState(FstState* s) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 479) assert(s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 480) return (s->val & 0b01000000) == 0b01000000; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 481) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 482) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 483) void fstStateSetStateNtrans(FstState* s, uint8_t n) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 484) assert(s->state == AnyTrans); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 485) if (n <= 0b00111111) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 486) s->val = (s->val & 0b11000000) | n; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 487) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 488) return; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 489) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 490) // state_ntrans -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 491) uint8_t fstStateStateNtrans(FstState* s, bool* null) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 492) assert(s->state == AnyTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 493) *null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 494) uint8_t n = s->val & 0b00111111; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 495) -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 496) if (n == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 497) *null = true; // None -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 498) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 499) return n; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 500) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 501) uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 502) assert(s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 503) uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 504) return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 505) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 506) uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 507) assert(s->state == AnyTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 508) return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 509) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 510) uint64_t fstStateNtransLen(FstState* s) { -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 511) assert(s->state == AnyTrans); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 512) bool null = false; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 513) fstStateStateNtrans(s, &null); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 514) return null == true ? 1 : 0; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 515) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 516) uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 517) bool null = false; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 518) uint8_t n = fstStateStateNtrans(s, &null); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 519) if (null != true) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 520) return n; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 521) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 522) int32_t len; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 523) uint8_t* data = fstSliceData(slice, &len); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 524) n = data[len - 2]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 525) // n = data[slice->end - 1]; // data[data.len() - 2] -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 526) return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 527) // then it is encoded in the state byte -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 528) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 529) Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 530) uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 531) if (oSizes == 0 || !fstStateIsFinalState(s)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 532) return 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 533) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 534) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 535) uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 536) - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 537) uint8_t* data = fstSliceData(slice, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 538) return unpackUint64(data + at, (uint8_t)oSizes); -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 539) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 540) uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 541) assert(s->state == AnyTrans); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 542) FstSlice* slice = &node->data; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 543) if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 544) uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 545) - fstStateTransIndexSize(s, node->version, node->nTrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 546) int32_t dlen = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 547) uint8_t* data = fstSliceData(slice, &dlen); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 548) uint64_t i = data[at + b]; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 549) // uint64_t i = slice->data[slice->start + at + b]; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 550) if (i >= node->nTrans) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 551) *null = true; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 552) } -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 553) return i; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 554) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 555) uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 556) - node->nTrans; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 557) uint64_t end = start + node->nTrans; -73d938b09a4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-08 18:07:13 +0800 558) FstSlice t = fstSliceCopy(slice, start, end - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 559) int32_t len = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 560) uint8_t* data = fstSliceData(&t, &len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 561) int i = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 562) for (; i < len; i++) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 563) uint8_t v = data[i]; -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 564) if (v == b) { -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 565) fstSliceDestroy(&t); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 566) return node->nTrans - i - 1; // bug -c5d53978743 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-24 18:41:00 +0800 567) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 568) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 569) if (i == len) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 570) *null = true; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 571) } -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 572) fstSliceDestroy(&t); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 573) } -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 574) -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 575) return 0; -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 576) } -169f6b3ad8c source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 23:53:45 +0800 577) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 578) // fst node function -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 579) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 580) FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 581) FstNode* n = (FstNode*)taosMemoryMalloc(sizeof(FstNode)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 582) if (n == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 583) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 584) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 585) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 586) FstState st = fstStateCreateFrom(slice, addr); -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 587) -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 588) if (st.state == EmptyFinal) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 589) n->data = fstSliceCreate(NULL, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 590) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 591) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 592) n->start = EMPTY_ADDRESS; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 593) n->end = EMPTY_ADDRESS; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 594) n->isFinal = true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 595) n->nTrans = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 596) n->sizes = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 597) n->finalOutput = 0; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 598) } else if (st.state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 599) n->data = fstSliceCopy(slice, 0, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 600) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 601) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 602) n->start = addr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 603) n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 604) n->isFinal = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 605) n->sizes = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 606) n->nTrans = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 607) n->finalOutput = 0; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 608) } else if (st.state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 609) FstSlice data = fstSliceCopy(slice, 0, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 610) PackSizes sz = fstStateSizes(&st, &data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 611) n->data = data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 612) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 613) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 614) n->start = addr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 615) n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 616) n->isFinal = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 617) n->nTrans = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 618) n->sizes = sz; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 619) n->finalOutput = 0; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 620) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 621) FstSlice data = fstSliceCopy(slice, 0, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 622) uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 623) uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 624) n->data = data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 625) n->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 626) n->state = st; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 627) n->start = addr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 628) n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 629) n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 630) n->nTrans = nTrans; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 631) n->sizes = sz; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 632) n->finalOutput = -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 633) fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 634) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 635) return n; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 636) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 637) -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 638) // debug state transition -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 639) static const char* fstNodeState(FstNode* node) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 640) FstState* st = &node->state; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 641) return fstStateStr[st->state]; -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 642) } -55282bbfa2f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 19:35:31 +0800 643) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 644) void fstNodeDestroy(FstNode* node) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 645) fstSliceDestroy(&node->data); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 646) taosMemoryFree(node); -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 647) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 648) FstTransitions* fstNodeTransitions(FstNode* node) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 649) FstTransitions* t = taosMemoryMalloc(sizeof(FstTransitions)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 650) if (NULL == t) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 651) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 652) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 653) FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 654) t->range = range; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 655) t->node = node; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 656) return t; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 657) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 658) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 659) // Returns the transition at index `i`. -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 660) bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 661) bool s = true; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 662) FstState* st = &node->state; -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 663) if (st->state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 664) trn->inp = fstStateInput(st, node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 665) trn->out = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 666) trn->addr = fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 667) } else if (st->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 668) trn->inp = fstStateInput(st, node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 669) trn->out = fstStateOutput(st, node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 670) trn->addr = fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 671) } else if (st->state == AnyTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 672) trn->inp = fstStateInputForAnyTrans(st, node, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 673) trn->out = fstStateOutputForAnyTrans(st, node, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 674) trn->addr = fstStateTransAddrForAnyTrans(st, node, i); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 675) } else { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 676) s = false; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 677) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 678) return s; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 679) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 680) -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 681) // Returns the transition address of the `i`th transition -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 682) bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 683) bool s = true; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 684) FstState* st = &node->state; -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 685) if (st->state == OneTransNext) { -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 686) assert(i == 0); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 687) fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 688) } else if (st->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 689) assert(i == 0); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 690) fstStateTransAddr(st, node); -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 691) } else if (st->state == AnyTrans) { -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 692) fstStateTransAddrForAnyTrans(st, node, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 693) } else if (FST_STATE_EMPTY_FINAL(node)) { -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 694) s = false; -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 695) } else { -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 696) assert(0); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 697) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 698) return s; -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 699) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 700) -dc9163a29a3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 12:13:44 +0800 701) // Finds the `i`th transition corresponding to the given input byte. -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 702) // If no transition for this byte exists, then `false` is returned. -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 703) bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 704) bool s = true; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 705) FstState* st = &node->state; -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 706) if (st->state == OneTransNext) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 707) if (fstStateInput(st, node) == b) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 708) *res = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 709) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 710) s = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 711) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 712) } else if (st->state == OneTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 713) if (fstStateInput(st, node) == b) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 714) *res = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 715) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 716) s = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 717) } -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 718) } else if (st->state == AnyTrans) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 719) bool null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 720) uint64_t out = fstStateFindInput(st, node, b, &null); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 721) if (null == false) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 722) *res = out; -2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 723) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 724) s = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 725) } -21592f572e5 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-25 17:39:53 +0800 726) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 727) return s; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 728) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 729) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 730) bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 731) size_t sz = taosArrayGetSize(builderNode->trans); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 732) assert(sz < 256); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 733) if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 734) return true; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 735) } else if (sz != 1 || builderNode->isFinal) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 736) fstStateCompileForAnyTrans(w, addr, builderNode); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 737) // AnyTrans->Compile(w, addr, node); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 738) } else { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 739) FstTransition* tran = taosArrayGet(builderNode->trans, 0); -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 740) if (tran->addr == lastAddr && tran->out == 0) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 741) fstStateCompileForOneTransNext(w, addr, tran->inp); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 742) // OneTransNext::compile(w, lastAddr, tran->inp); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 743) return true; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 744) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 745) fstStateCompileForOneTrans(w, addr, tran); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 746) // OneTrans::Compile(w, lastAddr, *tran); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 747) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 748) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 749) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 750) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 751) } -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 752) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 753) bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 754) return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 755) } -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 756) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 757) FstBuilder* fstBuilderCreate(void* w, FstType ty) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 758) FstBuilder* b = taosMemoryMalloc(sizeof(FstBuilder)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 759) if (NULL == b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 760) return b; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 761) } -32ac2d02c87 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:54:49 +0800 762) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 763) b->wrt = fstCountingWriterCreate(w); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 764) b->unfinished = fstUnFinishedNodesCreate(); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 765) b->registry = fstRegistryCreate(10000, 2); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 766) b->last = fstSliceCreate(NULL, 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 767) b->lastAddr = NONE_ADDRESS; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 768) b->len = 0; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 769) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 770) char buf64[8] = {0}; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 771) void* pBuf64 = buf64; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 772) taosEncodeFixedU64(&pBuf64, VERSION); -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 773) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 774) -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 775) pBuf64 = buf64; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 776) memset(buf64, 0, sizeof(buf64)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 777) taosEncodeFixedU64(&pBuf64, ty); -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 778) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -db2bf567c25 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 18:43:48 +0800 779) -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 780) return b; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 781) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 782) void fstBuilderDestroy(FstBuilder* b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 783) if (b == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 784) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 785) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 786) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 787) fstCountingWriterDestroy(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 788) fstUnFinishedNodesDestroy(b->unfinished); -260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 789) fstRegistryDestroy(b->registry); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 790) fstSliceDestroy(&b->last); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 791) taosMemoryFree(b); -260588b6922 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 23:07:07 +0800 792) } -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 793) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 794) bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 795) OrderType t = fstBuilderCheckLastKey(b, bs, true); -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 796) if (t == Ordered) { -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 797) // add log info -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 798) fstBuilderInsertOutput(b, bs, in); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 799) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 800) } -ebcb9be39fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-22 12:20:59 +0800 801) indexInfo("fst write key must be ordered"); -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 802) return false; -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 803) } -ccca561d11f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-23 09:52:29 +0800 804) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 805) void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 806) FstSlice* s = &bs; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 807) if (fstSliceIsEmpty(s)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 808) b->len = 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 809) fstUnFinishedNodesSetRootOutput(b->unfinished, in); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 810) return; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 811) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 812) // if (in != 0) { //if let Some(in) = in -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 813) // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 814) //} else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 815) // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 816) // out = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 817) //} -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 818) Output out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 819) uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 820) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 821) if (prefixLen == FST_SLICE_LEN(s)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 822) assert(out == 0); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 823) return; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 824) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 825) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 826) b->len += 1; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 827) fstBuilderCompileFrom(b, prefixLen); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 828) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 829) FstSlice sub = fstSliceCopy(s, prefixLen, s->end); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 830) fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 831) fstSliceDestroy(&sub); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 832) return; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 833) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 834) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 835) OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 836) FstSlice* input = &bs; -716c0045f8a source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 17:29:59 +0800 837) if (fstSliceIsEmpty(&b->last)) { -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 838) fstSliceDestroy(&b->last); -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 839) // deep copy or not -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 840) b->last = fstSliceDeepCopy(&bs, input->start, input->end); -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 841) } else { -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 842) int comp = fstSliceCompare(&b->last, &bs); -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 843) if (comp == 0 && ckDup) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 844) return DuplicateKey; -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 845) } else if (comp == 1) { -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 846) return OutOfOrdered; -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 847) } -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 848) // deep copy or not -d8d6c04fd50 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-09 16:18:35 +0800 849) fstSliceDestroy(&b->last); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 850) b->last = fstSliceDeepCopy(&bs, input->start, input->end); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 851) } -e769d0a0023 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 18:40:44 +0800 852) return Ordered; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 853) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 854) void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 855) CompiledAddr addr = NONE_ADDRESS; -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 856) while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 857) FstBuilderNode* bn = NULL; -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 858) if (addr == NONE_ADDRESS) { -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 859) bn = fstUnFinishedNodesPopEmpty(b->unfinished); -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 860) } else { -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 861) bn = fstUnFinishedNodesPopFreeze(b->unfinished, addr); -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 862) } -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 863) addr = fstBuilderCompile(b, bn); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 864) -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 865) fstBuilderNodeDestroy(bn); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 866) assert(addr != NONE_ADDRESS); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 867) // fstBuilderNodeDestroy(n); -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 868) } -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 869) fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 870) return; -5441bab5dc1 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 19:27:41 +0800 871) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 872) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 873) CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 874) if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 875) return EMPTY_ADDRESS; -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 876) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 877) FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 878) if (entry->state == FOUND) { -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 879) CompiledAddr ret = entry->addr; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 880) fstRegistryEntryDestroy(entry); -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 881) return ret; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 882) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 883) CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt)); -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 884) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 885) fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 886) b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 887) if (entry->state == NOTFOUND) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 888) FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 889) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 890) fstRegistryEntryDestroy(entry); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 891) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 892) return b->lastAddr; -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 893) } -0cd12fd353b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 12:05:03 +0800 894) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 895) void* fstBuilderInsertInner(FstBuilder* b) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 896) fstBuilderCompileFrom(b, 0); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 897) FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 898) CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 899) fstBuilderNodeDestroy(rootNode); -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 900) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 901) char buf64[8] = {0}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 902) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 903) void* pBuf64 = buf64; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 904) taosEncodeFixedU64(&pBuf64, b->len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 905) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 906) -d0844e5dda0 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-04 21:40:09 +0800 907) pBuf64 = buf64; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 908) taosEncodeFixedU64(&pBuf64, rootAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 909) fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 910) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 911) char buf32[4] = {0}; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 912) void* pBuf32 = buf32; -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 913) uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 914) taosEncodeFixedU32(&pBuf32, sum); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 915) fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 916) -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 917) fstCountingWriterFlush(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 918) // fstCountingWriterDestroy(b->wrt); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 919) // b->wrt = NULL; -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 920) return b->wrt; -e10b4bc05e0 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 00:16:42 +0800 921) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 922) void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 923) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 924) FstSlice fstNodeAsSlice(FstNode* node) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 925) FstSlice* slice = &node->data; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 926) FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 927) return s; -60e339b31df source/libs/index/src/index_fst.c (yihaoDeng 2021-11-20 13:13:21 +0800 928) } -980ace09b54 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-15 11:36:51 +0800 929) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 930) FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 931) FstLastTransition* trn = taosMemoryMalloc(sizeof(FstLastTransition)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 932) if (trn == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 933) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 934) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 935) -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 936) trn->inp = inp; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 937) trn->out = out; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 938) return trn; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 939) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 940) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 941) void fstLastTransitionDestroy(FstLastTransition* trn) { taosMemoryFree(trn); } -22938fcc5ed source/libs/index/src/index_fst.c (yihaoDeng 2022-02-23 17:51:07 +0800 942) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 943) void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 944) FstLastTransition* trn = unNode->last; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 945) if (trn == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 946) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 947) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 948) FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 949) taosArrayPush(unNode->node->trans, &t); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 950) fstLastTransitionDestroy(trn); -7e1f68f86fc source/libs/index/src/index_fst.c (yihaoDeng 2021-12-06 14:32:54 +0800 951) unNode->last = NULL; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 952) return; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 953) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 954) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 955) void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 956) if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 957) unNode->node->finalOutput += out; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 958) } -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 959) size_t sz = taosArrayGetSize(unNode->node->trans); -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 960) for (size_t i = 0; i < sz; i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 961) FstTransition* trn = taosArrayGet(unNode->node->trans, i); -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 962) trn->out += out; -06fe44cabdb source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 17:43:03 +0800 963) } -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 964) if (unNode->last) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 965) unNode->last->out += out; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 966) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 967) return; -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 968) } -0bbe42df45d source/libs/index/src/index_fst.c (yihaoDeng 2021-11-22 15:35:12 +0800 969) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 970) Fst* fstCreate(FstSlice* slice) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 971) int32_t slen; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 972) char* buf = fstSliceData(slice, &slen); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 973) if (slen < 36) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 974) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 975) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 976) uint64_t len = slen; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 977) uint64_t skip = 0; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 978) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 979) uint64_t version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 980) taosDecodeFixedU64(buf, &version); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 981) skip += sizeof(version); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 982) if (version == 0 || version > VERSION) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 983) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 984) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 985) -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 986) uint64_t type; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 987) taosDecodeFixedU64(buf + skip, &type); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 988) skip += sizeof(type); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 989) -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 990) uint32_t checkSum = 0; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 991) len -= sizeof(checkSum); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 992) taosDecodeFixedU32(buf + len, &checkSum); -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 993) if (taosCheckChecksum(buf, len, checkSum)) { -f9d83c044dd source/libs/index/src/index_fst.c (yihaoDeng 2022-01-10 23:42:58 +0800 994) indexError("index file is corrupted"); -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 995) // verify fst -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 996) return NULL; -15c0275333c source/libs/index/src/index_fst.c (yihaoDeng 2022-01-08 23:58:51 +0800 997) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 998) CompiledAddr rootAddr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 999) len -= sizeof(rootAddr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1000) taosDecodeFixedU64(buf + len, &rootAddr); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1001) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1002) uint64_t fstLen; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1003) len -= sizeof(fstLen); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1004) taosDecodeFixedU64(buf + len, &fstLen); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1005) // TODO(validate root addr) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1006) Fst* fst = (Fst*)taosMemoryCalloc(1, sizeof(Fst)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1007) if (fst == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1008) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1009) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1010) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1011) fst->meta = (FstMeta*)taosMemoryMalloc(sizeof(FstMeta)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1012) if (NULL == fst->meta) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1013) goto FST_CREAT_FAILED; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1014) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1015) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1016) fst->meta->version = version; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1017) fst->meta->rootAddr = rootAddr; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1018) fst->meta->ty = type; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1019) fst->meta->len = fstLen; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1020) fst->meta->checkSum = checkSum; -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1021) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1022) FstSlice* s = taosMemoryCalloc(1, sizeof(FstSlice)); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1023) *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice) - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1024) fst->data = s; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1025) -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1026) taosThreadMutexInit(&fst->mtx, NULL); -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1027) return fst; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1028) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1029) FST_CREAT_FAILED: -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1030) taosMemoryFree(fst->meta); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1031) taosMemoryFree(fst); -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1032) -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1033) return NULL; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1034) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1035) void fstDestroy(Fst* fst) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1036) if (fst) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1037) taosMemoryFree(fst->meta); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1038) fstSliceDestroy(fst->data); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1039) taosMemoryFree(fst->data); -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1040) taosThreadMutexDestroy(&fst->mtx); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1041) } -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1042) taosMemoryFree(fst); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1043) } -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1044) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1045) bool fstGet(Fst* fst, FstSlice* b, Output* out) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1046) // dec lock range -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1047) // taosThreadMutexLock(&fst->mtx); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1048) FstNode* root = fstGetRoot(fst); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1049) Output tOut = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1050) int32_t len; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1051) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1052) uint8_t* data = fstSliceData(b, &len); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1053) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1054) SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1055) taosArrayPush(nodes, &root); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1056) for (uint32_t i = 0; i < len; i++) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1057) uint8_t inp = data[i]; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1058) Output res = 0; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1059) if (false == fstNodeFindInput(root, inp, &res)) { -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1060) // taosThreadMutexUnlock(&fst->mtx); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1061) return false; -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1062) } -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1063) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1064) FstTransition trn; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1065) fstNodeGetTransitionAt(root, res, &trn); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1066) tOut += trn.out; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1067) root = fstGetNode(fst, trn.addr); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1068) taosArrayPush(nodes, &root); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1069) } -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1070) if (!FST_NODE_IS_FINAL(root)) { -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1071) // taosThreadMutexUnlock(&fst->mtx); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1072) return false; -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1073) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1074) tOut = tOut + FST_NODE_FINAL_OUTPUT(root); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1075) } -f8ca5dc15b2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 16:23:22 +0800 1076) -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1077) for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1078) FstNode** node = (FstNode**)taosArrayGet(nodes, i); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1079) fstNodeDestroy(*node); -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1080) } -29278327133 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-10 15:30:39 +0800 1081) taosArrayDestroy(nodes); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1082) // fst->root = NULL; -79057240bd6 source/libs/index/src/index_fst.c (afwerar 2022-03-20 00:47:45 +0800 1083) // taosThreadMutexUnlock(&fst->mtx); -2082e86476b source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 14:01:25 +0800 1084) *out = tOut; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1085) return true; -f07045c1d8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 17:28:58 +0800 1086) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1087) FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1088) // refactor later -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1089) return fstStreamBuilderCreate(fst, ctx); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1090) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1091) StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1092) if (sb == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1093) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1094) } -26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1095) return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); -26a98602263 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-15 23:24:03 +0800 1096) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1097) FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1098) // refactor later -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1099) return fstStreamBuilderCreate(fst, ctx); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1100) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1101) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1102) FstNode* fstGetRoot(Fst* fst) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1103) CompiledAddr rAddr = fstGetRootAddr(fst); -2eb0053e5ad source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 22:19:56 +0800 1104) return fstGetNode(fst, rAddr); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1105) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1106) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1107) FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1108) // refactor later -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1109) return fstNodeCreate(fst->meta->version, addr, fst->data); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1110) } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1111) FstType fstGetType(Fst* fst) { return fst->meta->ty; } -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1112) CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; } -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1113) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1114) Output fstEmptyFinalOutput(Fst* fst, bool* null) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1115) Output res = 0; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1116) FstNode* node = fstGetRoot(fst); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1117) if (FST_NODE_IS_FINAL(node)) { -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1118) *null = false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1119) res = FST_NODE_FINAL_OUTPUT(node); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1120) } else { -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1121) *null = true; -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1122) } -40fc4b6cf38 source/libs/index/src/index_fst.c (yihaoDeng 2022-01-02 17:50:40 +0800 1123) fstNodeDestroy(node); -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1124) return res; -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1125) } -500130daf70 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 18:34:24 +0800 1126) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1127) bool fstVerify(Fst* fst) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1128) uint32_t len, checkSum = fst->meta->checkSum; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1129) uint8_t* data = fstSliceData(fst->data, &len); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1130) TSCKSUM initSum = 0; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1131) if (!taosCheckChecksumWhole(data, len)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1132) return false; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1133) } -a09dcbdd85f source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 10:59:51 +0800 1134) return true; -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1135) } -c0ca718eed2 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-26 11:29:43 +0800 1136) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1137) // data bound function -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1138) FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1139) FstBoundWithData* b = taosMemoryCalloc(1, sizeof(FstBoundWithData)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1140) if (b == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1141) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1142) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1143) -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1144) if (data != NULL) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1145) b->data = fstSliceCopy(data, data->start, data->end); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1146) } else { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1147) b->data = fstSliceCreate(NULL, 0); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1148) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1149) b->type = type; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1150) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1151) return b; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1152) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1153) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1154) bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1155) int comp = fstSliceCompare(slice, &bound->data); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1156) if (bound->type == Included) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1157) return comp > 0 ? true : false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1158) } else if (bound->type == Excluded) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1159) return comp >= 0 ? true : false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1160) } else { -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1161) return false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1162) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1163) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1164) bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1165) if (bound->type == Unbounded) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1166) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1167) } else { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1168) return fstSliceIsEmpty(&bound->data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1169) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1170) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1171) -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1172) bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { return bound->type == Excluded ? false : true; } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1173) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1174) void fstBoundDestroy(FstBoundWithData* bound) { taosMemoryFree(bound); } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1175) -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1176) StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1177) FstBoundWithData* max) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1178) StreamWithState* sws = taosMemoryCalloc(1, sizeof(StreamWithState)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1179) if (sws == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1180) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1181) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1182) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1183) sws->fst = fst; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1184) sws->aut = automation; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1185) sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1186) -0c5f2d1da95 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 21:41:41 +0800 1187) sws->emptyOutput.null = true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1188) sws->emptyOutput.out = 0; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1189) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1190) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1191) sws->endAt = max; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1192) streamWithStateSeekMin(sws, min); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1193) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1194) return sws; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1195) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1196) void streamWithStateDestroy(StreamWithState* sws) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1197) if (sws == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1198) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1199) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1200) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1201) taosArrayDestroy(sws->inp); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1202) taosArrayDestroyEx(sws->stack, streamStateDestroy); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1203) -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1204) taosMemoryFree(sws); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1205) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1206) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1207) bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1208) AutomationCtx* aut = sws->aut; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1209) if (fstBoundWithDataIsEmpty(min)) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1210) if (fstBoundWithDataIsIncluded(min)) { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1211) sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1212) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1213) StreamState s = {.node = fstGetRoot(sws->fst), -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1214) .trans = 0, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1215) .out = {.null = false, .out = 0}, -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1216) .autState = automFuncs[aut->type].start(aut)}; // auto.start callback -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1217) taosArrayPush(sws->stack, &s); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1218) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1219) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1220) FstSlice* key = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1221) bool inclusize = false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1222) -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1223) if (min->type == Included) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1224) key = &min->data; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1225) inclusize = true; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1226) } else if (min->type == Excluded) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1227) key = &min->data; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1228) } else { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1229) return false; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1230) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1231) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1232) FstNode* node = fstGetRoot(sws->fst); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1233) Output out = 0; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1234) // void* autState = sws->aut->start(); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1235) void* autState = automFuncs[aut->type].start(aut); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1236) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1237) int32_t len; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1238) uint8_t* data = fstSliceData(key, &len); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1239) for (uint32_t i = 0; i < len; i++) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1240) uint8_t b = data[i]; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1241) uint64_t res = 0; -2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1242) bool find = fstNodeFindInput(node, b, &res); -2354cd9ec0e source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 22:21:51 +0800 1243) if (find == true) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1244) FstTransition trn; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1245) fstNodeGetTransitionAt(node, res, &trn); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1246) void* preState = autState; -cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1247) // autState = sws->aut->accept(preState, b); -cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1248) autState = automFuncs[aut->type].accept(aut, preState, b); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1249) taosArrayPush(sws->inp, &b); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1250) StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1251) taosArrayPush(sws->stack, &s); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1252) out += trn.out; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1253) node = fstGetNode(sws->fst, trn.addr); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1254) } else { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1255) // This is a little tricky. We're in this case if the -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1256) // given bound is not a prefix of any key in the FST. -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1257) // Since this is a minimum bound, we need to find the -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1258) // first transition in this node that proceeds the current -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1259) // input byte. -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1260) FstTransitions* trans = fstNodeTransitions(node); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1261) uint64_t i = 0; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1262) for (i = trans->range.start; i < trans->range.end; i++) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1263) FstTransition trn; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1264) if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1265) break; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1266) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1267) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1268) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1269) StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1270) taosArrayPush(sws->stack, &s); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1271) return true; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1272) } -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1273) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1274) uint32_t sz = taosArrayGetSize(sws->stack); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1275) if (sz != 0) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1276) StreamState* s = taosArrayGet(sws->stack, sz - 1); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1277) if (inclusize) { -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1278) s->trans -= 1; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1279) taosArrayPop(sws->inp); -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1280) } else { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1281) FstNode* n = s->node; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1282) uint64_t trans = s->trans; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1283) FstTransition trn; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1284) fstNodeGetTransitionAt(n, trans - 1, &trn); -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1285) StreamState s = { -236b8bc3c8d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-30 17:03:00 +0800 1286) .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; -23c9ea680b3 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-29 23:37:21 +0800 1287) taosArrayPush(sws->stack, &s); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1288) return true; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1289) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1290) return false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1291) } -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1292) -23bef711fca source/libs/index/src/index_fst.c (Shuduo Sang 2022-03-15 21:35:04 +0800 1293) return false; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1294) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1295) StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1296) AutomationCtx* aut = sws->aut; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1297) FstOutput output = sws->emptyOutput; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1298) if (output.null == false) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1299) FstSlice emptySlice = fstSliceCreate(NULL, 0); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1300) if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1301) taosArrayDestroyEx(sws->stack, streamStateDestroy); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1302) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1303) return NULL; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1304) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1305) void* start = automFuncs[aut->type].start(aut); -cd653e56abd source/libs/index/src/index_fst.c (yihaoDeng 2021-12-11 15:42:09 +0800 1306) if (automFuncs[aut->type].isMatch(aut, start)) { -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1307) FstSlice s = fstSliceCreate(NULL, 0); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1308) return swsResultCreate(&s, output, callback == NULL ? NULL : callback(start)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1309) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1310) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1311) SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1312) while (taosArrayGetSize(sws->stack) > 0) { -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1313) StreamState* p = (StreamState*)taosArrayPop(sws->stack); -f0110af30ec source/libs/index/src/index_fst.c (yihaoDeng 2021-12-16 22:04:47 +0800 1314) if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1315) if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1316) taosArrayPop(sws->inp); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1317) } -6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1318) streamStateDestroy(p); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1319) continue; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1320) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1321) FstTransition trn; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1322) fstNodeGetTransitionAt(p->node, p->trans, &trn); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1323) -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1324) Output out = p->out.out + trn.out; -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1325) void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1326) void* tState = (callback == NULL) ? NULL : callback(nextState); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1327) bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); -f45119ea5e2 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-26 15:09:21 +0800 1328) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1329) FstNode* nextNode = fstGetNode(sws->fst, trn.addr); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1330) taosArrayPush(nodes, &nextNode); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1331) taosArrayPush(sws->inp, &(trn.inp)); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1332) -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1333) if (FST_NODE_IS_FINAL(nextNode)) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1334) // void *eofState = sws->aut->acceptEof(nextState); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1335) void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1336) if (eofState != NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1337) isMatch = automFuncs[aut->type].isMatch(aut, eofState); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1338) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1339) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1340) StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1341) taosArrayPush(sws->stack, &s1); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1342) -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1343) StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1344) taosArrayPush(sws->stack, &s2); -6f3c49ee73d source/libs/index/src/index_fst.c (yihaoDeng 2021-12-14 16:17:38 +0800 1345) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1346) size_t isz = taosArrayGetSize(sws->inp); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1347) uint8_t* buf = (uint8_t*)taosMemoryMalloc(isz * sizeof(uint8_t)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1348) for (uint32_t i = 0; i < isz; i++) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1349) buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1350) } -00000000000 source/libs/index/src/indexFst.c (Not Committed Yet 2022-04-02 15:19:56 +0800 1351) FstSlice slice = fstSliceCreate(buf, isz); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1352) if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1353) taosArrayDestroyEx(sws->stack, streamStateDestroy); -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1354) sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1355) taosMemoryFreeClear(buf); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1356) fstSliceDestroy(&slice); -06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1357) taosArrayDestroy(nodes); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1358) return NULL; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1359) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1360) if (FST_NODE_IS_FINAL(nextNode) && isMatch) { -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1361) FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1362) StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1363) taosMemoryFreeClear(buf); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1364) fstSliceDestroy(&slice); -06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1365) taosArrayDestroy(nodes); -6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1366) nodes = NULL; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1367) return result; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1368) } -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1369) taosMemoryFreeClear(buf); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1370) fstSliceDestroy(&slice); -6a4199d3ede source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:01:05 +0800 1371) }; -06535139684 source/libs/index/src/index_fst.c (yihaoDeng 2022-03-29 15:06:28 +0800 1372) taosArrayDestroy(nodes); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1373) return NULL; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1374) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1375) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1376) StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1377) StreamWithStateResult* result = taosMemoryCalloc(1, sizeof(StreamWithStateResult)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1378) if (result == NULL) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1379) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1380) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1381) -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1382) result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1383) result->out = fOut; -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1384) result->state = state; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1385) return result; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1386) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1387) void swsResultDestroy(StreamWithStateResult* result) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1388) if (NULL == result) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1389) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1390) } -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1391) -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1392) fstSliceDestroy(&result->data); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1393) startWithStateValueDestroy(result->state); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1394) taosMemoryFree(result); -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1395) } -e3ccb28c097 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-02 10:32:58 +0800 1396) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1397) void streamStateDestroy(void* s) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1398) if (NULL == s) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1399) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1400) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1401) StreamState* ss = (StreamState*)s; -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1402) fstNodeDestroy(ss->node); -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1403) } -fbb9d515bf6 source/libs/index/src/index_fst.c (yihaoDeng 2021-11-30 14:51:24 +0800 1404) -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1405) FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1406) FstStreamBuilder* b = taosMemoryCalloc(1, sizeof(FstStreamBuilder)); -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1407) if (NULL == b) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1408) return NULL; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1409) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1410) -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1411) b->fst = fst; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1412) b->aut = aut; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1413) b->min = fstBoundStateCreate(Unbounded, NULL); -5d4d7b47a0f source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 16:36:31 +0800 1414) b->max = fstBoundStateCreate(Unbounded, NULL); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1415) return b; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1416) } -10b70175b5b source/libs/index/src/index_fst.c (yihaoDeng 2021-12-21 22:57:30 +0800 1417) void fstStreamBuilderDestroy(FstStreamBuilder* b) { -7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1418) fstSliceDestroy(&b->min->data); -7ee1cf62ca4 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 12:00:20 +0800 1419) fstSliceDestroy(&b->max->data); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1420) taosMemoryFreeClear(b->min); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1421) taosMemoryFreeClear(b->max); -222db126bcf source/libs/index/src/index_fst.c (afwerar 2022-03-26 00:29:53 +0800 1422) taosMemoryFree(b); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1423) } -30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1424) void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1425) if (b == NULL) { -30d49687e9a source/libs/index/src/index_fst.c (yihaoDeng 2022-03-28 19:01:57 +0800 1426) return; -9233663a985 source/libs/index/src/index_fst.c (yihaoDeng 2022-02-22 22:02:28 +0800 1427) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1428) if (type == GE) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1429) b->min->type = Included; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1430) fstSliceDestroy(&(b->min->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1431) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1432) } else if (type == GT) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1433) b->min->type = Excluded; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1434) fstSliceDestroy(&(b->min->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1435) b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1436) } else if (type == LE) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1437) b->max->type = Included; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1438) fstSliceDestroy(&(b->max->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1439) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1440) } else if (type == LT) { -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1441) b->max->type = Excluded; -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1442) fstSliceDestroy(&(b->max->data)); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1443) b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1444) } -d46ca756bf3 source/libs/index/src/index_fst.c (yihaoDeng 2021-12-01 00:24:12 +0800 1445) } From ba5503da680deacb72ec4b543ff777365956f601 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Apr 2022 03:27:07 -0400 Subject: [PATCH 14/42] insert using implement --- include/libs/nodes/querynodes.h | 1 - include/util/tdef.h | 2 +- source/libs/command/src/command.c | 13 +++- source/libs/parser/inc/parInsertData.h | 7 +- source/libs/parser/src/parInsert.c | 19 ++++-- source/libs/parser/src/parInsertData.c | 71 +++++++++++--------- source/libs/parser/test/parserInsertTest.cpp | 15 ++++- 7 files changed, 81 insertions(+), 47 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index dec709535f..ca4646a5ca 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -277,7 +277,6 @@ typedef struct SVnodeModifOpStmt { ENodeType nodeType; ENodeType sqlNodeType; SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint32_t insertType; // insert data from [file|sql statement| bound statement] const char* sql; // current sql statement position diff --git a/include/util/tdef.h b/include/util/tdef.h index b1bebcee46..be88afcf66 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -307,7 +307,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_DAYS_PER_FILE (1 * 1440) // unit minute +#define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 7ffbd4f80a..4d4ac6c1e4 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -16,7 +16,16 @@ #include "command.h" #include "tdatablock.h" -// #define SET_VARSTR(pData, val, pOffset) +static int32_t getSchemaBytes(const SSchema* pSchema) { + switch (pSchema->type) { + case TSDB_DATA_TYPE_BINARY: + return (pSchema->bytes - VARSTR_HEADER_SIZE); + case TSDB_DATA_TYPE_NCHAR: + return (pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + default: + return pSchema->bytes; + } +} static void buildRspData(const STableMeta* pMeta, char* pData) { int32_t* pColSizes = (int32_t*)pData; @@ -50,7 +59,7 @@ static void buildRspData(const STableMeta* pMeta, char* pData) { // Length pData += BitmapLen(numOfRows); for (int32_t i = 0; i < numOfRows; ++i) { - *(int32_t*)pData = pMeta->schema[i].bytes; + *(int32_t*)pData = getSchemaBytes(pMeta->schema + i); pData += sizeof(int32_t); } pColSizes[2] = sizeof(int32_t) * numOfRows; diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index 5894fc88cc..ee17de50e0 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -77,7 +77,7 @@ typedef struct STableDataBlocks { STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; bool cloned; - + int32_t createTbReqLen; SParsedDataColInfo boundColumnInfo; SRowBuilder rowBuilder; } STableDataBlocks; @@ -118,6 +118,7 @@ static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks* pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? dataBuf->pTableMeta->uid : dataBuf->pTableMeta->suid); pBlocks->uid = dataBuf->pTableMeta->uid; pBlocks->sversion = dataBuf->pTableMeta->sversion; + pBlocks->schemaLen = dataBuf->createTbReqLen; if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -136,7 +137,7 @@ void destroyBlockHashmap(SHashObj* pDataBlockHash); int initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks); + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d7eb31b6cd..d9cf0b39b5 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -753,7 +753,7 @@ static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, S } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision, const SName* pName) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const SName* pName) { if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -763,9 +763,9 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pCxt->tags.boundColumns[i]]; - param.schema = pSchema; - CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 + param.schema = pTagSchema; + CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); @@ -791,6 +791,7 @@ static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, S if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + pBackup->uid = tGenIdPI64(); return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } @@ -833,7 +834,11 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_NK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision, &name)); + CHECK_CODE(parseTagsClause(pCxt, pCxt->pTableMeta->schema, getTableInfo(pCxt->pTableMeta).precision, &name)); + NEXT_TOKEN(pCxt->pSql, sToken); + if (TK_NK_RP != sToken.type) { + return buildSyntaxErrMsg(&pCxt->msg, ") is expected", sToken.z); + } return TSDB_CODE_SUCCESS; } @@ -1015,7 +1020,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { STableDataBlocks *dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) @@ -1046,7 +1051,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // merge according to vgId if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } return buildOutput(pCxt); } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index f70e514b5a..088b25d544 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -149,8 +149,28 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star return TSDB_CODE_SUCCESS; } +static int32_t buildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { + int32_t len = tSerializeSVCreateTbReq(NULL, pCreateTbReq); + if (pBlocks->nAllocSize - pBlocks->size < len) { + pBlocks->nAllocSize += len + pBlocks->rowSize; + char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); + if (pTmp != NULL) { + pBlocks->pData = pTmp; + memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + } else { + pBlocks->nAllocSize -= len + pBlocks->rowSize; + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + char* pBuf = pBlocks->pData + pBlocks->size; + tSerializeSVCreateTbReq((void**)&pBuf, pCreateTbReq); + pBlocks->size += len; + pBlocks->createTbReqLen = len; + return TSDB_CODE_SUCCESS; +} + int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -163,6 +183,13 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 return ret; } + if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctbCfg.pTag) { + ret = buildCreateTbMsg(*dataBlocks, pCreateTbReq); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + } + taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES); if (pBlockList) { taosArrayPush(pBlockList, dataBlocks); @@ -294,7 +321,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey int32_t extendedRowSize = getExtendedRowSize(dataBuf); SBlockKeyTuple *pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char * pBlockData = pBlocks->data; + char * pBlockData = pBlocks->data + pBlocks->schemaLen; int n = 0; while (n < nRows) { pBlkKeyTuple->skey = TD_ROW_KEY((STSRow *)pBlockData); @@ -340,44 +367,26 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey } // Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, int8_t schemaAttached, bool isRawPayload) { +static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, bool isRawPayload) { // TODO: optimize this function, handle the case while binary is not presented STableMeta* pTableMeta = pTableDataBlock->pTableMeta; STableComInfo tinfo = getTableInfo(pTableMeta); SSchema* pSchema = getTableColumnSchema(pTableMeta); + int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk)); - pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk); + memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); + pDataBlock = (char*)pDataBlock + nonDataLen; int32_t flen = 0; // original total length of row - - // schema needs to be included into the submit data block - if (schemaAttached) { - int32_t numOfCols = getNumOfColumns(pTableDataBlock->pTableMeta); - for(int32_t j = 0; j < numOfCols; ++j) { - STColumn* pCol = (STColumn*) pDataBlock; - pCol->colId = htons(pSchema[j].colId); - pCol->type = pSchema[j].type; - pCol->bytes = htons(pSchema[j].bytes); - pCol->offset = 0; - - pDataBlock = (char*)pDataBlock + sizeof(STColumn); + if (isRawPayload) { + for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { flen += TYPE_BYTES[pSchema[j].type]; } - - int32_t schemaSize = sizeof(STColumn) * numOfCols; - pBlock->schemaLen = schemaSize; - } else { - if (isRawPayload) { - for (int32_t j = 0; j < tinfo.numOfColumns; ++j) { - flen += TYPE_BYTES[pSchema[j].type]; - } - } - pBlock->schemaLen = 0; } + pBlock->schemaLen = pTableDataBlock->createTbReqLen; - char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); + char* p = pTableDataBlock->pData + nonDataLen; pBlock->dataLen = 0; int32_t numOfRows = pBlock->numOfRows; @@ -414,7 +423,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB return pBlock->dataLen + pBlock->schemaLen; } -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { +int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** pVgDataBlocks) { const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); @@ -429,7 +438,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); @@ -474,7 +483,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); + int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index 90e2ba3db2..d292fcf8b0 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -62,7 +62,7 @@ protected: void dumpReslut() { SVnodeModifOpStmt* pStmt = getVnodeModifStmt(res_); size_t num = taosArrayGetSize(pStmt->pDataBlocks); - cout << "schemaAttache:" << (int32_t)pStmt->schemaAttache << ", payloadType:" << (int32_t)pStmt->payloadType << ", insertType:" << pStmt->insertType << ", numOfVgs:" << num << endl; + cout << "payloadType:" << (int32_t)pStmt->payloadType << ", insertType:" << pStmt->insertType << ", numOfVgs:" << num << endl; for (size_t i = 0; i < num; ++i) { SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, i); cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; @@ -81,7 +81,6 @@ protected: void checkReslut(int32_t numOfTables, int16_t numOfRows1, int16_t numOfRows2 = -1) { SVnodeModifOpStmt* pStmt = getVnodeModifStmt(res_); - ASSERT_EQ(pStmt->schemaAttache, 0); ASSERT_EQ(pStmt->payloadType, PAYLOAD_TYPE_KV); ASSERT_EQ(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); size_t num = taosArrayGetSize(pStmt->pDataBlocks); @@ -168,6 +167,18 @@ TEST_F(InsertTest, multiTableMultiRowTest) { checkReslut(2, 3, 2); } +// INSERT INTO +// tb1_name USING st1_name [(tag1_name, ...)] TAGS (tag1_value, ...) VALUES (field1_value, ...) +// tb2_name USING st2_name [(tag1_name, ...)] TAGS (tag1_value, ...) VALUES (field1_value, ...) +TEST_F(InsertTest, autoCreateTableTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 using st1 tags(1, 'wxy') values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 3); +} + TEST_F(InsertTest, toleranceTest) { setDatabase("root", "test"); From 11653686420b652b35a83d48d928bae7c245ee1b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 2 Apr 2022 15:39:20 +0800 Subject: [PATCH 15/42] retentions param bug fix and deliver to vnode --- source/common/src/tmsg.c | 6 +++--- source/dnode/mgmt/vm/src/vmMsg.c | 5 +++++ source/dnode/mnode/impl/src/mndDb.c | 3 ++- source/dnode/vnode/inc/tsdb.h | 3 ++- source/libs/parser/src/parTranslater.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2a16b58565..988230bd3f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -367,7 +367,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx)); - for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); @@ -376,7 +376,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols); pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); @@ -408,7 +408,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols); buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx)); - for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) { + for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma); buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); diff --git a/source/dnode/mgmt/vm/src/vmMsg.c b/source/dnode/mgmt/vm/src/vmMsg.c index c3ad74d246..a5e8fe6088 100644 --- a/source/dnode/mgmt/vm/src/vmMsg.c +++ b/source/dnode/mgmt/vm/src/vmMsg.c @@ -30,6 +30,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->tsdbCfg.retentions = pCreate->pRetensions; pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; pCfg->walCfg.level = pCreate->walLevel; @@ -70,6 +71,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); if (pVnode != NULL) { + tFreeSCreateVnodeReq(&createReq); dDebug("vgId:%d, already exist", createReq.vgId); vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; @@ -88,12 +90,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { + tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); return -1; } int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); if (code != 0) { + tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); @@ -103,6 +107,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { code = vmWriteVnodesToFile(pMgmt); if (code != 0) { + tFreeSCreateVnodeReq(&createReq); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); terrno = code; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 462f0eb85a..44c547bec3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -69,7 +69,8 @@ void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, + sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) goto DB_ENCODE_OVER; int32_t dataPos = 0; diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 0d3fcffe7d..b287a1fe4c 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -55,13 +55,14 @@ typedef struct STsdbCfg { int8_t precision; int8_t update; int8_t compression; - uint64_t lruCacheSize; int32_t daysPerFile; int32_t minRowsPerFileBlock; int32_t maxRowsPerFileBlock; int32_t keep; int32_t keep1; int32_t keep2; + uint64_t lruCacheSize; + SArray *retentions; } STsdbCfg; // query condition to build multi-table data block iterator diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 773060beab..09f56f1657 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -938,7 +938,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe SNode* pNode = NULL; int32_t index = 0; FOREACH(pNode, pRetentions) { - if (0 == index % 2) { + if (0 == ((index++) & 1)) { pFreq = (SValueNode*)pNode; } else { pKeep = (SValueNode*)pNode; @@ -951,6 +951,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe taosArrayPush(pReq->pRetensions, &retention); } } + pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions); } return TSDB_CODE_SUCCESS; } From dfac724697c05146b007a48f8dd21f98f490e52f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 16:33:17 +0800 Subject: [PATCH 16/42] handle except --- source/libs/index/src/indexFst.c | 9 +++++++++ source/libs/index/test/fstTest.cc | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index a146564c9e..bc3ecea7a5 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -642,6 +642,9 @@ static const char* fstNodeState(FstNode* node) { } void fstNodeDestroy(FstNode* node) { + if (node == NULL) { + return; + } fstSliceDestroy(&node->data); taosMemoryFree(node); } @@ -1247,7 +1250,10 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); + StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; + node = NULL; + taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); @@ -1271,6 +1277,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { return true; } } + + fstNodeDestroy(node); + uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { StreamState* s = taosArrayGet(sws->stack, sz - 1); diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index c45f655746..bfe1bb21bf 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -480,6 +480,15 @@ void checkFstCheckIteratorRange2() { assert(result.size() == 4); automCtxDestroy(ctx); } + { + // range search + std::vector result; + AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS); + // [b, e) + m->SearchRange(ctx, "bb", GE, "ed", LT, result); + assert(result.size() == 3); + automCtxDestroy(ctx); + } { // range search std::vector result; @@ -635,11 +644,11 @@ int main(int argc, char* argv[]) { // path suid colName ver // iterTFileReader(argv[1], argv[2], argv[3], argv[4]); //} - // checkFstCheckIterator1(); - // checkFstCheckIterator2(); - // checkFstCheckIteratorPrefix(); - // checkFstCheckIteratorRange1(); - // checkFstCheckIteratorRange2(); + checkFstCheckIterator1(); + checkFstCheckIterator2(); + checkFstCheckIteratorPrefix(); + checkFstCheckIteratorRange1(); + checkFstCheckIteratorRange2(); checkFstCheckIteratorRange3(); // checkFstLongTerm(); // checkFstPrefixSearch(); From bf08e3482df22f080254506edbf866b3fe7bf8c1 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 2 Apr 2022 16:45:43 +0800 Subject: [PATCH 17/42] [add cases] --- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/db/create_all_options.sim | 486 ++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 tests/script/tsim/db/create_all_options.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d8fbfce55f..b756976394 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -5,6 +5,7 @@ ./test.sh -f tsim/user/basic1.sim # ---- db +./test.sh -f tsim/db/create_all_options.sim ./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim new file mode 100644 index 0000000000..7f39474f4d --- /dev/null +++ b/tests/script/tsim/db/create_all_options.sim @@ -0,0 +1,486 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$loop_cnt = 0 +check_dnode_ready_1: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data00 != 1 then + return -1 +endi +if $data01 != localhost:7100 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready_1 +endi +if $data14 != ready then + goto check_dnode_ready_1 +endi +if $data24 != ready then + goto check_dnode_ready_1 +endi + +print ============= create database with all options +#database_option: { +# | BLOCKS value [3~1000, default: 6] +# | CACHE value [default: 16] +# | CACHELAST value [0, 1, 2, 3, default: 0] +# | COMP [0 | 1 | 2, default: 2] +# | DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +# | FSYNC value [0 ~ 180000 ms, default: 3000] +# | MAXROWS value [200~10000, default: 4096] +# | MINROWS value [10~1000, default: 100] +# | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] +# | PRECISION ['ms' | 'us' | 'ns', default: ms] +# | QUORUM value [1 | 2, default: 1] +# | REPLICA value [1 | 3, default: 1] +# | TTL value [1d ~ , default: 1] +# | WAL value [1 | 2, default: 1] +# | VGROUPS value [default: 2] +# | SINGLE_STABLE [0 | 1, default: ] +# | STREAM_MODE [0 | 1, default: ] +# +#$data0_db : name +#$data1_db : create_time +#$data2_db : vgroups +#$data3_db : ntables +#$data4_db : replica +#$data5_db : quorum +#$data6_db : days +#$data7_db : keep +#$data8_db : cache +#$data9_db : blocks +#$data10_db : minrows +#$data11_db : maxrows +#$data12_db : wal +#$data13_db : fsync +#$data14_db : comp +#$data15_db : cachelast +#$data16_db : precision + +print ====> create database db, with default +sql create database db +sql show databases +print rows: $rows +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $rows != 2 then + return -1 +endi +if $data0_db != db then # name + return -1 +endi +if $data2_db != 2 then # vgroups + return -1 +endi +if $data3_db != 0 then # ntables + return -1 +endi +if $data4_db != 1 then # replica + return -1 +endi +if $data5_db != 1 then # quorum + return -1 +endi +if $data6_db != 14400 then # days + return -1 +endi +if $data7_db != 5256000,5256000,5256000 then # keep + return -1 +endi +if $data8_db != 16 then # cache + return -1 +endi +if $data9_db != 6 then # blocks + return -1 +endi +if $data10_db != 100 then # minrows + return -1 +endi +if $data11_db != 4096 then # maxrows + return -1 +endi +if $data12_db != 1 then # wal + return -1 +endi +if $data13_db != 3000 then # fsync + return -1 +endi +if $data14_db != 2 then # comp + return -1 +endi +if $data15_db != 0 then # cachelast + return -1 +endi +if $data16_db != ms then # precision + return -1 +endi +sql drop database db + +print ====> BLOCKS value [3~1000, default: 6] +sql create database db BLOCKS 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data9_db != 3 then + return -1 +endi +sql drop database db + +sql create database db BLOCKS 1000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data9_db != 1000 then + return -1 +endi +sql drop database db +sql_error create database db BLOCKS 2 +sql_error create database db BLOCKS 0 +sql_error create database db BLOCKS -1 + +print ====> CACHE value [default: 16] +sql create database db CACHE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data8_db != 1 then + return -1 +endi +sql drop database db + +sql create database db CACHE 128 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data8_db != 128 then + return -1 +endi +sql drop database db + +print ====> CACHELAST value [0, 1, 2, 3, default: 0] +sql create database db CACHELAST 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 1 then + return -1 +endi +sql drop database db + +sql create database db CACHELAST 2 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 2 then + return -1 +endi +sql drop database db + +sql create database db CACHELAST 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data15_db != 3 then + return -1 +endi +sql drop database db +sql_error create database db CACHELAST 4 +sql_error create database db CACHELAST -1 + +print ====> COMP [0 | 1 | 2, default: 2] +sql create database db COMP 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data14_db != 1 then + return -1 +endi +sql drop database db + +sql create database db COMP 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data14_db != 0 then + return -1 +endi +sql drop database db +sql_error create database db COMP 3 +sql_error create database db COMP -1 + +#print ====> DAYS value [60m ~ min(3650d,keep), default: 10d, unit may be minut/hour/day] +#print ====> KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] +#sql create database db DAYS 60m KEEP 60m +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 60 then +# return -1 +#endi +#if $data7_db != 60,60,60 then +# return -1 +#endi +#sql drop database db +#sql create database db DAYS 60m KEEP 1d +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 60 then +# return -1 +#endi +#if $data7_db != 1440,1440,1440 then +# return -1 +#endi +#sql create database db DAYS 3650d KEEP 365000d +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data6_db != 5256000 then +# return -1 +#endi +#if $data7_db != 525600000,525600000,525600000 then +# return -1 +#endi +#sql drop database db +#sql_error create database db DAYS -59m +#sql_error create database db DAYS 59m +#sql_error create database db DAYS 5256001m +#sql_error create database db DAYS 3651d +#sql_error create database db KEEP -59m +#sql_error create database db KEEP 14399m +#sql_error create database db KEEP 525600001m +#sql_error create database db KEEP 365001d + +print ====> FSYNC value [0 ~ 180000 ms, default: 3000] +sql create database db FSYNC 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data13_db != 0 then + return -1 +endi +sql drop database db + +sql create database db FSYNC 180000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data13_db != 180000 then + return -1 +endi +sql drop database db +sql_error create database db FSYNC 180001 +sql_error create database db FSYNC -1 + +print ====> MAXROWS value [200~10000, default: 4096], MINROWS value [10~1000, default: 100] +sql create database db MAXROWS 10000 MINROWS 1000 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data10_db != 1000 then + return -1 +endi +if $data11_db != 10000 then + return -1 +endi +sql drop database db + +sql create database db MAXROWS 200 MINROWS 10 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data10_db != 10 then + return -1 +endi +if $data11_db != 200 then + return -1 +endi +sql drop database db +sql_error create database db MAXROWS -1 +sql_error create database db MAXROWS 0 +sql_error create database db MAXROWS 199 +sql_error create database db MAXROWS 10001 +sql_error create database db MINROWS -1 +sql_error create database db MINROWS 0 +sql_error create database db MINROWS 9 +sql_error create database db MINROWS 1001 +sql_error create database db MAXROWS 500 MINROWS 1000 + +print ====> PRECISION ['ms' | 'us' | 'ns', default: ms] +sql create database db PRECISION 'us' +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data16_db != us then + return -1 +endi +sql drop database db + +sql create database db PRECISION 'ns' +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data16_db != ns then + return -1 +endi +sql drop database db +sql_error create database db PRECISION 'as' +sql_error create database db PRECISION -1 + +print ====> QUORUM value [1 | 2, default: 1] +#sql create database db QUORUM 2 +#sql show databases +#print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $data5_db != 2 then +# return -1 +#endi +#sql drop database db + +sql create database db QUORUM 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data5_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db QUORUM 3 +sql_error create database db QUORUM 0 +sql_error create database db QUORUM -1 + +print ====> REPLICA value [1 | 3, default: 1] +sql create database db REPLICA 3 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data4_db != 3 then + return -1 +endi +sql drop database db + +sql create database db REPLICA 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data4_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db REPLICA 2 +sql_error create database db REPLICA 0 +sql_error create database db REPLICA -1 +sql_error create database db REPLICA 4 + +print ====> TTL value [1d ~ , default: 1] +sql create database db TTL 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db TTL 10 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXX_db != 10 then +# return -1 +#endi +sql drop database db +sql_error create database db TTL 0 +sql_error create database db TTL -1 + +print ====> WAL value [1 | 2, default: 1] +sql create database db WAL 2 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data12_db != 2 then + return -1 +endi +sql drop database db + +sql create database db WAL 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data12_db != 1 then + return -1 +endi +sql drop database db +sql_error create database db WAL 3 +sql_error create database db WAL -1 +sql_error create database db WAL 0 + +print ====> VGROUPS value [1~4096, default: 2] +sql create database db VGROUPS 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data2_db != 1 then + return -1 +endi +sql drop database db + +sql create database db VGROUPS 16 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +if $data2_db != 16 then + return -1 +endi +sql drop database db +sql_error create database db VGROUPS 4097 +sql_error create database db VGROUPS -1 +sql_error create database db VGROUPS 0 + +print ====> SINGLE_STABLE [0 | 1, default: ] +sql create database db SINGLE_STABLE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db SINGLE_STABLE 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXXX_db != 0 then +# return -1 +#endi +sql drop database db +sql_error create database db SINGLE_STABLE 2 +sql_error create database db SINGLE_STABLE -1 + +print ====> STREAM_MODE [0 | 1, default: ] +sql create database db STREAM_MODE 1 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXX_db != 1 then +# return -1 +#endi +sql drop database db + +sql create database db STREAM_MODE 0 +sql show databases +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db +#if $dataXXX_db != 0 then +# return -1 +#endi +sql drop database db +sql_error create database db STREAM_MODE 2 +sql_error create database db STREAM_MODE -1 + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From 20f1e3f5e516fde2f0f2a388498da1ff5b219907 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 17:22:19 +0800 Subject: [PATCH 18/42] refact dnode --- include/libs/transport/trpc.h | 7 +++++-- source/dnode/mgmt/bm/src/bmWorker.c | 2 +- source/dnode/mgmt/main/inc/dnd.h | 2 -- source/dnode/mgmt/main/src/dndExec.c | 2 +- source/dnode/mgmt/main/src/dndMsg.c | 2 +- source/dnode/mgmt/main/src/dndTransport.c | 25 ++++++++++------------- source/dnode/mgmt/mm/src/mmWorker.c | 4 ++-- source/dnode/mgmt/qm/src/qmWorker.c | 2 +- source/dnode/mgmt/vm/src/vmWorker.c | 4 ++-- 9 files changed, 24 insertions(+), 26 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 157e0cb721..3e2f596784 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -58,6 +58,9 @@ typedef struct { void *pNode; } SNodeMsg; +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); +typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + typedef struct SRpcInit { uint16_t localPort; // local port char * label; // for debug purpose @@ -74,10 +77,10 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + RpcCfp cfp; // call back to retrieve the client auth info, for server app only - int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + RpcAfp afp;; void *parent; } SRpcInit; diff --git a/source/dnode/mgmt/bm/src/bmWorker.c b/source/dnode/mgmt/bm/src/bmWorker.c index 932c008e34..a5a97f6af0 100644 --- a/source/dnode/mgmt/bm/src/bmWorker.c +++ b/source/dnode/mgmt/bm/src/bmWorker.c @@ -18,7 +18,7 @@ static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rpcRsp); + tmsgSendRsp(&rpcRsp); dTrace("msg:%p, is freed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index f3a409f257..cf33787108 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -166,8 +166,6 @@ int32_t dndInitClient(SDnode *pDnode); void dndCleanupClient(SDnode *pDnode); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); #ifdef __cplusplus diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index 8837da6d48..3bbdb1a686 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -78,7 +78,7 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); if (pRpc->msgType & 1U) { SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } dTrace("msg:%p, is freed", pMsg); diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index bb0f8ccb89..76b66467bf 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -86,7 +86,7 @@ _OVER: dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); if (pRpc->msgType & 1U) { SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } dTrace("msg:%p, is freed", pMsg); taosFreeQitem(pMsg); diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index d463f2ba7a..f863db2a0c 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -36,8 +36,7 @@ static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, S dndProcessRpcMsg(pWrapper, pMsg, pEpSet); } -static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { - SDnode *pDnode = parent; +static void dndProcessResponse(SDnode *pDnode, SRpcMsg *pRsp, SEpSet *pEpSet) { STransMgmt *pMgmt = &pDnode->trans; tmsg_t msgType = pRsp->msgType; @@ -69,7 +68,7 @@ int32_t dndInitClient(SDnode *pDnode) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND"; rpcInit.numOfThreads = 1; - rpcInit.cfp = dndProcessResponse; + rpcInit.cfp = (RpcCfp)dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; @@ -101,8 +100,7 @@ void dndCleanupClient(SDnode *pDnode) { } } -static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { - SDnode *pDnode = param; +static void dndProcessRequest(SDnode *pDnode, SRpcMsg *pReq, SEpSet *pEpSet) { STransMgmt *pMgmt = &pDnode->trans; tmsg_t msgType = pReq->msgType; @@ -179,10 +177,8 @@ static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *e return code; } -static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SDnode *pDnode = parent; - - if (dndGetHideUserAuth(parent, user, spi, encrypt, secret, ckey) == 0) { +static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { + if (dndGetHideUserAuth(pDnode, user, spi, encrypt, secret, ckey) == 0) { dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } @@ -244,11 +240,11 @@ int32_t dndInitServer(SDnode *pDnode) { rpcInit.localPort = pDnode->serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = dndProcessRequest; + rpcInit.cfp = (RpcCfp)dndProcessRequest; rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - rpcInit.afp = dndRetrieveUserAuthInfo; + rpcInit.afp = (RpcAfp)dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; pMgmt->serverRpc = rpcOpen(&rpcInit); @@ -363,6 +359,7 @@ int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + memcpy(pHead, pReq, sizeof(SRpcMsg)); memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); @@ -372,7 +369,7 @@ int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) } } -void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { +static void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { if (pWrapper->procType != PROC_CHILD) { dndSendRpcRsp(pWrapper, pRsp); } else { @@ -380,7 +377,7 @@ void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { } } -void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { +static void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { if (pWrapper->procType != PROC_CHILD) { rpcRegisterBrokenLinkArg(pMsg); } else { @@ -406,4 +403,4 @@ SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { .releaseHandleFp = dndReleaseHandle, }; return msgCb; -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/mm/src/mmWorker.c b/source/dnode/mgmt/mm/src/mmWorker.c index c4aafe05e4..cde4eb853a 100644 --- a/source/dnode/mgmt/mm/src/mmWorker.c +++ b/source/dnode/mgmt/mm/src/mmWorker.c @@ -37,7 +37,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); } SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } @@ -60,7 +60,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { if (pRpc->handle != NULL && code != 0) { dError("msg:%p, failed to process since %s", pMsg, terrstr()); SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle}; - dndSendRsp(pMgmt->pWrapper, &rsp); + tmsgSendRsp(&rsp); } } diff --git a/source/dnode/mgmt/qm/src/qmWorker.c b/source/dnode/mgmt/qm/src/qmWorker.c index 14efb311b1..5ceb9dbf88 100644 --- a/source/dnode/mgmt/qm/src/qmWorker.c +++ b/source/dnode/mgmt/qm/src/qmWorker.c @@ -18,7 +18,7 @@ static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { diff --git a/source/dnode/mgmt/vm/src/vmWorker.c b/source/dnode/mgmt/vm/src/vmWorker.c index 4be6311cf8..c228e6e7dd 100644 --- a/source/dnode/mgmt/vm/src/vmWorker.c +++ b/source/dnode/mgmt/vm/src/vmWorker.c @@ -18,7 +18,7 @@ static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; - dndSendRsp(pWrapper, &rsp); + tmsgSendRsp(&rsp); } static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { @@ -116,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp); if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; - dndSendRsp(pVnode->pWrapper, pRsp); + tmsgSendRsp(pRsp); taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; From a651affa47a71a3e1befd9b75107f251ed7d5e43 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 2 Apr 2022 17:47:27 +0800 Subject: [PATCH 19/42] [TD-13756]: log file open fail error. --- source/util/src/tlog.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1f26ab7c51..563acfec78 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -214,6 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; + taosSsleep(3); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); @@ -347,16 +348,12 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (pFile == NULL) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - TdFilePtr pOldFile = tsLogObj.logHandle->pFile; - tsLogObj.logHandle->pFile = pFile; - taosUnLockLogFile(pOldFile); - taosCloseFile(&pOldFile); taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines From eb18ad240b45a34d6e06714c6644e7e4f8f36d7e Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 2 Apr 2022 17:54:29 +0800 Subject: [PATCH 20/42] [TD-13756]: log file open fail error. --- source/os/src/osFile.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 1b73f96896..a7855539a4 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -325,7 +325,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -365,7 +365,7 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = pread(pFile->fd, buf, count, offset); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t nleft = count; int64_t nwritten = 0; @@ -414,7 +414,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t ret = lseek(pFile->fd, (long)offset, whence); #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); @@ -429,7 +429,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); @@ -456,7 +456,7 @@ int32_t taosLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif @@ -469,7 +469,7 @@ int32_t taosUnLockFile(TdFilePtr pFile) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif @@ -529,7 +529,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. return ftruncate(pFile->fd, l_size); #endif @@ -750,7 +750,7 @@ void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { if (pFile == NULL) { return NULL; } - assert(pFile->fd >= 0); + assert(pFile->fd >= 0); // Please check if you have closed the file. void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); return ptr; From 2eb30ee0f0a6e486a6b3c87471b2e4d7ca4f8b8c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 18:08:38 +0800 Subject: [PATCH 21/42] refact transport --- source/dnode/mgmt/dm/inc/dmInt.h | 1 + source/dnode/mgmt/dm/src/dmInt.c | 14 +-- source/dnode/mgmt/dm/src/dmMsg.c | 4 +- source/dnode/mgmt/main/inc/dnd.h | 12 +-- source/dnode/mgmt/main/inc/dndInt.h | 1 - source/dnode/mgmt/main/src/dndExec.c | 63 ----------- source/dnode/mgmt/main/src/dndInt.c | 1 + source/dnode/mgmt/main/src/dndTransport.c | 125 ++++++++++++++++------ 8 files changed, 106 insertions(+), 115 deletions(-) diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index 3036d1f5ad..79d7e926ce 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -33,6 +33,7 @@ typedef struct SDnodeMgmt { SRWLatch latch; SSingleWorker mgmtWorker; SSingleWorker statusWorker; + SMsgCb msgCb; const char *path; SDnode *pDnode; SMgmtWrapper *pWrapper; diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index 3c5f394d5e..8c46c402ab 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -112,17 +112,14 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return -1; } - if (dndInitServer(pDnode) != 0) { - dError("failed to init trans server since %s", terrstr()); - return -1; - } - - if (dndInitClient(pDnode) != 0) { - dError("failed to init trans client since %s", terrstr()); + if (dndInitTrans(pDnode) != 0) { + dError("failed to init transport since %s", terrstr()); return -1; } pWrapper->pMgmt = pMgmt; + pMgmt->msgCb = dndCreateMsgcb(pWrapper); + dInfo("dnode-mgmt is initialized"); return 0; } @@ -151,8 +148,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); + dndCleanupTrans(pDnode); dInfo("dnode-mgmt is cleaned up"); } diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index 3795cffbfa..02a288e2fd 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -57,7 +57,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { pMgmt->statusSent = 1; dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); - dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg); + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt, &epSet); + tmsgSendReq(&pMgmt->msgCb, &epSet, &rpcMsg); } static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index cf33787108..d948aa305c 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -92,6 +92,7 @@ typedef struct SMgmtWrapper { char *path; int32_t refCount; SRWLatch latch; + ENodeType ntype; bool deployed; bool required; EProcType procType; @@ -160,13 +161,10 @@ const char *dndNodeProcStr(ENodeType ntype); const char *dndEventStr(EDndEvent ev); // dndTransport.h -int32_t dndInitServer(SDnode *pDnode); -void dndCleanupServer(SDnode *pDnode); -int32_t dndInitClient(SDnode *pDnode); -void dndCleanupClient(SDnode *pDnode); -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); -int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); -SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +int32_t dndInitTrans(SDnode *pDnode); +void dndCleanupTrans(SDnode *pDnode); +SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 0c4ada1df3..b4d9de5184 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -49,7 +49,6 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndTransport.c int32_t dndInitMsgHandle(SDnode *pDnode); -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); // dndFile.c TdFilePtr dndCheckRunning(const char *dataDir); diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index 3bbdb1a686..a98c0954fe 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -65,53 +65,6 @@ void dndCloseNode(SMgmtWrapper *pWrapper) { dDebug("node:%s, mgmt has been closed", pWrapper->name); } -static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - SRpcMsg *pRpc = &pMsg->rpcMsg; - pRpc->pCont = pCont; - dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); - - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - int32_t code = (*msgFp)(pWrapper, pMsg); - - if (code != 0) { - dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - tmsgSendRsp(&rsp); - } - - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pCont); - } -} - -static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, - ProcFuncType ftype) { - pMsg->pCont = pCont; - dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); - - switch (ftype) { - case PROC_REGIST: - rpcRegisterBrokenLinkArg(pMsg); - break; - case PROC_RELEASE: - rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); - rpcFreeCont(pCont); - break; - case PROC_REQ: - dndSendReqToMnode(pWrapper, pMsg); - // dndSendReq(pWrapper, (const SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); - break; - case PROC_RSP: - dndSendRpcRsp(pWrapper, pMsg); - break; - default: - break; - } - taosMemoryFree(pMsg); -} static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { char tstr[8] = {0}; @@ -135,22 +88,6 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { return 0; } -static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .shm = pWrapper->shm, - .pParent = pWrapper, - .name = pWrapper->name}; - return cfg; -} static int32_t dndRunInSingleProcess(SDnode *pDnode) { dInfo("dnode run in single process"); diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 25e4d89c5b..5e4b0a1f7b 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -95,6 +95,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { pWrapper->path = strdup(path); pWrapper->shm.id = -1; pWrapper->pDnode = pDnode; + pWrapper->ntype = n; if (pWrapper->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index f863db2a0c..cefa77890d 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -61,7 +61,7 @@ static void dndProcessResponse(SDnode *pDnode, SRpcMsg *pRsp, SEpSet *pEpSet) { } } -int32_t dndInitClient(SDnode *pDnode) { +static int32_t dndInitClient(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; SRpcInit rpcInit; @@ -91,7 +91,7 @@ int32_t dndInitClient(SDnode *pDnode) { return 0; } -void dndCleanupClient(SDnode *pDnode) { +static void dndCleanupClient(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; if (pMgmt->clientRpc) { rpcClose(pMgmt->clientRpc); @@ -227,7 +227,7 @@ static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, ch return rpcRsp.code; } -int32_t dndInitServer(SDnode *pDnode) { +static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); @@ -257,7 +257,7 @@ int32_t dndInitServer(SDnode *pDnode) { return 0; } -void dndCleanupServer(SDnode *pDnode) { +static void dndCleanupServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; if (pMgmt->serverRpc) { rpcClose(pMgmt->serverRpc); @@ -266,6 +266,17 @@ void dndCleanupServer(SDnode *pDnode) { } } +int32_t dndInitTrans(SDnode *pDnode) { + if (dndInitServer(pDnode) != 0) return -1; + if (dndInitClient(pDnode) != 0) return -1; + return 0; +} + +void dndCleanupTrans(SDnode *pDnode) { + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); +} + int32_t dndInitMsgHandle(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->trans; @@ -315,46 +326,28 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *p return 0; } -int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - STransMgmt *pTrans = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper2 = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper2 != NULL) { - dmGetMnodeEpSet(pWrapper2->pMgmt, &epSet); - dndReleaseWrapper(pWrapper2); - } - return dndSendRpcReq(pTrans, &epSet, pReq); -} - -void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { +static void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) { if (pRsp->code == TSDB_CODE_APP_NOT_READY) { - SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE); - if (pDnodeWrapper != NULL) { - dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp); - dndReleaseWrapper(pDnodeWrapper); - } else { - rpcSendResponse(pRsp); + if (pWrapper->ntype == MNODE) { + dmSendRedirectRsp(pWrapper->pMgmt, pRsp); + return; } - } else { - rpcSendResponse(pRsp); } + + rpcSendResponse(pRsp); } -int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { - SDnode *pDnode = pWrapper->pDnode; - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { +static int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) { + if (dndGetStatus(pWrapper->pDnode) != DND_STAT_RUNNING) { terrno = TSDB_CODE_DND_OFFLINE; dError("failed to send rpc msg since %s, handle:%p", terrstr(), pReq->handle); return -1; } if (pWrapper->procType != PROC_CHILD) { - return dndSendRpcReq(&pDnode->trans, pEpSet, pReq); + return dndSendRpcReq(&pWrapper->pDnode->trans, pEpSet, pReq); } else { - int32_t headLen = sizeof(SRpcMsg) + sizeof(SEpSet); - char *pHead = taosMemoryMalloc(headLen); + char *pHead = taosMemoryMalloc(sizeof(SRpcMsg) + sizeof(SEpSet)); if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -362,8 +355,8 @@ int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) memcpy(pHead, pReq, sizeof(SRpcMsg)); memcpy(pHead + sizeof(SRpcMsg), pEpSet, sizeof(SEpSet)); - - taosProcPutToParentQ(pWrapper->pProc, pReq, headLen, pReq->pCont, pReq->contLen, PROC_REQ); + taosProcPutToParentQ(pWrapper->pProc, pHead, sizeof(SRpcMsg) + sizeof(SEpSet), pReq->pCont, pReq->contLen, + PROC_REQ); taosMemoryFree(pHead); return 0; } @@ -404,3 +397,67 @@ SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) { }; return msgCb; } + +static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + SRpcMsg *pRpc = &pMsg->rpcMsg; + pRpc->pCont = pCont; + dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle); + + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + int32_t code = (*msgFp)(pWrapper, pMsg); + + if (code != 0) { + dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + dndSendRsp(pWrapper, &rsp); + } + + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pCont); + } +} + +static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, + ProcFuncType ftype) { + pMsg->pCont = pCont; + dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle); + + switch (ftype) { + case PROC_REGIST: + rpcRegisterBrokenLinkArg(pMsg); + break; + case PROC_RELEASE: + rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code); + rpcFreeCont(pCont); + break; + case PROC_REQ: + dndSendRpcReq(&pWrapper->pDnode->trans, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); + break; + case PROC_RSP: + dndSendRpcRsp(pWrapper, pMsg); + break; + default: + break; + } + taosMemoryFree(pMsg); +} + +SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { + SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .shm = pWrapper->shm, + .pParent = pWrapper, + .name = pWrapper->name}; + return cfg; +} \ No newline at end of file From 781e869e4e716f72503665cc37bc3a78d20f24ae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 18:11:45 +0800 Subject: [PATCH 22/42] handle except --- source/libs/transport/test/rclient.c | 45 +++++++++++++++------------- source/libs/transport/test/rserver.c | 7 ++++- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 7d3c3aa012..cdf0908167 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -23,20 +23,20 @@ #include "tutil.h" typedef struct { - int index; - SEpSet epSet; - int num; - int numOfReqs; - int msgSize; - tsem_t rspSem; - tsem_t * pOverSem; + int index; + SEpSet epSet; + int num; + int numOfReqs; + int msgSize; + tsem_t rspSem; + tsem_t * pOverSem; TdThread thread; - void * pRpc; + void * pRpc; } SInfo; static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; - tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, - pMsg->code); + // tError("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, + // pMsg->code); if (pEpSet) pInfo->epSet = *pEpSet; @@ -51,9 +51,9 @@ static void *sendRequest(void *param) { SInfo * pInfo = (SInfo *)param; SRpcMsg rpcMsg = {0}; - tDebug("thread:%d, start to send request", pInfo->index); + tError("thread:%d, start to send request", pInfo->index); - tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + tError("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); int u100 = 0; int u500 = 0; int u1000 = 0; @@ -68,7 +68,7 @@ static void *sendRequest(void *param) { // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); - if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); + if (pInfo->num % 20000 == 0) tError("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); int64_t end = taosGetTimestampUs() - start; @@ -88,7 +88,7 @@ static void *sendRequest(void *param) { } tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); - tDebug("thread:%d, it is over", pInfo->index); + tError("thread:%d, it is over", pInfo->index); tcount++; return NULL; @@ -104,7 +104,7 @@ int main(int argc, char *argv[]) { char secret[20] = "mypassword"; struct timeval systemTime; int64_t startTime, endTime; - TdThreadAttr thattr; + TdThreadAttr thattr; // server info epSet.inUse = 0; @@ -124,7 +124,7 @@ int main(int argc, char *argv[]) { rpcInit.ckey = "key"; rpcInit.spi = 1; rpcInit.connType = TAOS_CONN_CLIENT; - rpcDebugFlag = 143; + rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -170,6 +170,10 @@ int main(int argc, char *argv[]) { } } + const char *path = "/tmp/transport/client"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); @@ -178,8 +182,8 @@ int main(int argc, char *argv[]) { return -1; } - tInfo("client is initialized"); - tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); + tError("client is initialized"); + tError("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); taosGetTimeOfDay(&systemTime); startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; @@ -208,8 +212,9 @@ int main(int argc, char *argv[]) { endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; float usedTime = (endTime - startTime) / 1000.0f; // mseconds - tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); + tError("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tError("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, + msgSize); int ch = getchar(); UNUSED(ch); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 3a086371b0..8ed3bbc960 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -125,7 +125,7 @@ int main(int argc, char *argv[]) { rpcInit.idleTime = 2 * 1500; rpcInit.afp = retrieveAuthInfo; - rpcDebugFlag = 143; + rpcDebugFlag = 131; for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { @@ -160,6 +160,11 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; + + const char *path = "/tmp/transport/server"; + taosRemoveDir(path); + taosMkDir(path); + tstrncpy(tsLogDir, path, PATH_MAX); taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); From 8bcdf8e200132cf066cd3a8b3253f3eecd33c413 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 2 Apr 2022 18:34:17 +0800 Subject: [PATCH 23/42] [TD-14241]: refactor string functions --- source/libs/function/src/builtins.c | 1 + source/libs/scalar/src/sclfunc.c | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 126117f982..99a247b45e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -536,6 +536,7 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) { paraType = pParam->node.resType.type; } pFunc->node.resType = (SDataType) { .bytes = paraBytes, .type = paraType }; + break; //int32_t paraTypeFirst, totalBytes = 0, sepBytes = 0; //int32_t firstParamIndex = 0; //if (pFunc->funcType == FUNCTION_TYPE_CONCAT_WS) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index bc508f6e22..c02729a084 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -333,15 +333,15 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t inputLen = 0; int32_t numOfRows = 0; for (int32_t i = 0; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { + return TSDB_CODE_FAILED; + } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } } for (int32_t i = 0; i < inputNum; ++i) { - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || - GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { - return TSDB_CODE_FAILED; - } pInputData[i] = pInput[i].columnData; input[i] = pInputData[i]->pData; if (pInput[i].numOfRows == 1) { @@ -402,20 +402,21 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t inputLen = 0; int32_t numOfRows = 0; - for (int32_t i = 0; i < inputNum; ++i) { + for (int32_t i = 1; i < inputNum; ++i) { + if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || + GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[1])) { + return TSDB_CODE_FAILED; + } if (pInput[i].numOfRows > numOfRows) { numOfRows = pInput[i].numOfRows; } } for (int32_t i = 0; i < inputNum; ++i) { - if (!IS_VAR_DATA_TYPE(GET_PARAM_TYPE(&pInput[i])) || - GET_PARAM_TYPE(&pInput[i]) != GET_PARAM_TYPE(&pInput[0])) { - return TSDB_CODE_FAILED; - } pInputData[i] = pInput[i].columnData; if (i == 0) { // calculate required separator space - inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2); + int32_t factor = (GET_PARAM_TYPE(&pInput[1]) == TSDB_DATA_TYPE_NCHAR) ? TSDB_NCHAR_SIZE : 1; + inputLen += (pInputData[0]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows * (inputNum - 2) * factor; } else if (pInput[i].numOfRows == 1) { inputLen += (pInputData[i]->varmeta.length - VARSTR_HEADER_SIZE) * numOfRows; } else { @@ -429,7 +430,6 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *output = outputBuf; for (int32_t k = 0; k < numOfRows; ++k) { - char *sep = pInputData[0]->pData; if (colDataIsNull_s(pInputData[0], k)) { colDataAppendNULL(pOutputData, k); continue; @@ -437,7 +437,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int16_t dataLen = 0; for (int32_t i = 1; i < inputNum; ++i) { - if (colDataIsNull_f(pInputData[i]->nullbitmap, k)) { + if (colDataIsNull_s(pInputData[i], k)) { continue; } @@ -449,6 +449,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p if (i < inputNum - 1) { //insert the separator + char *sep = pInputData[0]->pData; memcpy(varDataVal(output) + dataLen, varDataVal(sep), varDataLen(sep)); dataLen += varDataLen(sep); } From 745fa09bbb903f042755cefb446f3661cb1b793a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 18:58:28 +0800 Subject: [PATCH 24/42] refact transport --- source/dnode/mgmt/bm/inc/bm.h | 31 --------------- source/dnode/mgmt/bm/inc/bmInt.h | 2 +- source/dnode/mgmt/dm/inc/dm.h | 38 ------------------- source/dnode/mgmt/dm/inc/dmInt.h | 3 +- source/dnode/mgmt/dm/src/dmMsg.c | 1 - source/dnode/mgmt/dm/src/dmWorker.c | 5 --- source/dnode/mgmt/main/inc/dnd.h | 30 +++++++++++++++ source/dnode/mgmt/main/inc/dndInt.h | 10 ----- source/dnode/mgmt/main/src/dndStr.c | 2 +- source/dnode/mgmt/main/src/dndTransport.c | 37 ++++-------------- source/dnode/mgmt/mm/inc/mm.h | 35 ----------------- source/dnode/mgmt/mm/inc/mmInt.h | 2 +- source/dnode/mgmt/mm/src/mmInt.c | 8 ---- source/dnode/mgmt/qm/inc/qm.h | 31 --------------- source/dnode/mgmt/qm/inc/qmInt.h | 2 +- source/dnode/mgmt/sm/inc/sm.h | 31 --------------- source/dnode/mgmt/sm/inc/smInt.h | 2 +- source/dnode/mgmt/vm/inc/vm.h | 46 ----------------------- source/dnode/mgmt/vm/inc/vmInt.h | 2 +- 19 files changed, 45 insertions(+), 273 deletions(-) delete mode 100644 source/dnode/mgmt/bm/inc/bm.h delete mode 100644 source/dnode/mgmt/dm/inc/dm.h delete mode 100644 source/dnode/mgmt/mm/inc/mm.h delete mode 100644 source/dnode/mgmt/qm/inc/qm.h delete mode 100644 source/dnode/mgmt/sm/inc/sm.h delete mode 100644 source/dnode/mgmt/vm/inc/vm.h diff --git a/source/dnode/mgmt/bm/inc/bm.h b/source/dnode/mgmt/bm/inc/bm.h deleted file mode 100644 index 79cf76d113..0000000000 --- a/source/dnode/mgmt/bm/inc/bm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_BNODE_H_ -#define _TD_DND_BNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void bmGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_BNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/bm/inc/bmInt.h b/source/dnode/mgmt/bm/inc/bmInt.h index f19ba4e034..7a719633fa 100644 --- a/source/dnode/mgmt/bm/inc/bmInt.h +++ b/source/dnode/mgmt/bm/inc/bmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_BNODE_INT_H_ #define _TD_DND_BNODE_INT_H_ -#include "bm.h" +#include "dnd.h" #include "bnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/dm/inc/dm.h b/source/dnode/mgmt/dm/inc/dm.h deleted file mode 100644 index 3984e6dbd4..0000000000 --- a/source/dnode/mgmt/dm/inc/dm.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_DNODE_H_ -#define _TD_DND_DNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SDnodeMgmt SDnodeMgmt; - -void dmGetMgmtFp(SMgmtWrapper *pWrapper); -void dmInitMsgHandles(SMgmtWrapper *pWrapper); - -void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); -void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_DNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index 79d7e926ce..a8f46ef7e2 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_DNODE_INT_H_ #define _TD_DND_DNODE_INT_H_ -#include "dm.h" +#include "dnd.h" #ifdef __cplusplus extern "C" { @@ -45,6 +45,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); // dmMsg.c +void dmInitMsgHandles(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index 02a288e2fd..90117044f9 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "dmInt.h" -#include "vm.h" void dmSendStatusReq(SDnodeMgmt *pMgmt) { SDnode *pDnode = pMgmt->pDnode; diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/src/dmWorker.c index 63b9704b78..9045a44232 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/src/dmWorker.c @@ -14,12 +14,7 @@ */ #define _DEFAULT_SOURCE -#include "bm.h" #include "dmInt.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index d948aa305c..b324a279bc 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -165,7 +165,37 @@ int32_t dndInitTrans(SDnode *pDnode); void dndCleanupTrans(SDnode *pDnode); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); +int32_t dndInitMsgHandle(SDnode *pDnode); +// mgmt +void dmGetMgmtFp(SMgmtWrapper *pWrapper); +void bmGetMgmtFp(SMgmtWrapper *pWrapper); +void qmGetMgmtFp(SMgmtWrapper *pMgmt); +void smGetMgmtFp(SMgmtWrapper *pWrapper); +void vmGetMgmtFp(SMgmtWrapper *pWrapper); +void mmGetMgmtFp(SMgmtWrapper *pMgmt); + +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); + +typedef struct { + int32_t openVnodes; + int32_t totalVnodes; + int32_t masterNum; + int64_t numOfSelectReqs; + int64_t numOfInsertReqs; + int64_t numOfInsertSuccessReqs; + int64_t numOfBatchInsertReqs; + int64_t numOfBatchInsertSuccessReqs; +} SVnodesStat; + +void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); +int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); +void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); +int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, + SMonGrantInfo *pGrantInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index b4d9de5184..5ef52e3942 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -18,13 +18,6 @@ #include "dnd.h" -#include "bm.h" -#include "dm.h" -#include "mm.h" -#include "qm.h" -#include "sm.h" -#include "vm.h" - #ifdef __cplusplus extern "C" { #endif @@ -47,9 +40,6 @@ void dndHandleEvent(SDnode *pDnode, EDndEvent event); void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); -// dndTransport.c -int32_t dndInitMsgHandle(SDnode *pDnode); - // dndFile.c TdFilePtr dndCheckRunning(const char *dataDir); int32_t dndReadShmFile(SDnode *pDnode); diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/src/dndStr.c index 8a5af68b4f..caaa96de59 100644 --- a/source/dnode/mgmt/main/src/dndStr.c +++ b/source/dnode/mgmt/main/src/dndStr.c @@ -25,7 +25,7 @@ const char *dndStatStr(EDndStatus status) { case DND_STAT_STOPPED: return "stopped"; default: - return "unknown"; + return "UNKNOWN"; } } diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index cefa77890d..2be2da4e46 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -142,20 +142,14 @@ static void dndProcessRequest(SDnode *pDnode, SRpcMsg *pReq, SEpSet *pEpSet) { } } -static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) { - STransMgmt *pMgmt = &pDnode->trans; - SEpSet epSet = {0}; - - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); - dndReleaseWrapper(pWrapper); - } - - rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp); +static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { + SEpSet epSet = {0}; + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmGetMnodeEpSet(pWrapper->pMgmt, &epSet); + rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); } -static int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { +static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { int32_t code = 0; char pass[TSDB_PASSWORD_LEN + 1] = {0}; @@ -183,21 +177,6 @@ static int32_t dndRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *spi, ch return 0; } - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE); - if (pWrapper != NULL) { - if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) { - dndReleaseWrapper(pWrapper); - dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); - return 0; - } - dndReleaseWrapper(pWrapper); - } - - if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from mnode since %s", terrstr()); - return -1; - } - SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); @@ -285,11 +264,9 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; - int32_t vgId = pWrapper->msgVgIds[msgIndex]; + int8_t vgId = pWrapper->msgVgIds[msgIndex]; if (msgFp == NULL) continue; - // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); - SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; if (vgId == QND_VGID) { if (pHandle->pQndWrapper != NULL) { diff --git a/source/dnode/mgmt/mm/inc/mm.h b/source/dnode/mgmt/mm/inc/mm.h deleted file mode 100644 index 6ed6c42d93..0000000000 --- a/source/dnode/mgmt/mm/inc/mm.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_MNODE_H_ -#define _TD_DND_MNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void mmGetMgmtFp(SMgmtWrapper *pMgmt); - -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey); -int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_MNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mm/inc/mmInt.h b/source/dnode/mgmt/mm/inc/mmInt.h index 86cba97a33..48c11ae8c4 100644 --- a/source/dnode/mgmt/mm/inc/mmInt.h +++ b/source/dnode/mgmt/mm/inc/mmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_MNODE_INT_H_ #define _TD_DND_MNODE_INT_H_ -#include "mm.h" +#include "dnd.h" #include "mnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/src/mmInt.c index 301ca598e6..bd7446780d 100644 --- a/source/dnode/mgmt/mm/src/mmInt.c +++ b/source/dnode/mgmt/mm/src/mmInt.c @@ -241,14 +241,6 @@ void mmGetMgmtFp(SMgmtWrapper *pWrapper) { pWrapper->fp = mgmtFp; } -int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - - int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey); - dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt); - return code; -} - int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, SMonGrantInfo *pGrantInfo) { SMnodeMgmt *pMgmt = pWrapper->pMgmt; diff --git a/source/dnode/mgmt/qm/inc/qm.h b/source/dnode/mgmt/qm/inc/qm.h deleted file mode 100644 index e28ea3e948..0000000000 --- a/source/dnode/mgmt/qm/inc/qm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_QNODE_H_ -#define _TD_DND_QNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void qmGetMgmtFp(SMgmtWrapper *pMgmt); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_QNODE_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/qm/inc/qmInt.h b/source/dnode/mgmt/qm/inc/qmInt.h index 3e975663d3..b4d57488ef 100644 --- a/source/dnode/mgmt/qm/inc/qmInt.h +++ b/source/dnode/mgmt/qm/inc/qmInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_QNODE_INT_H_ #define _TD_DND_QNODE_INT_H_ -#include "qm.h" +#include "dnd.h" #include "qnode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/sm/inc/sm.h b/source/dnode/mgmt/sm/inc/sm.h deleted file mode 100644 index 82a52e5d1f..0000000000 --- a/source/dnode/mgmt/sm/inc/sm.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_SNODE_H_ -#define _TD_DND_SNODE_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -void smGetMgmtFp(SMgmtWrapper *pWrapper); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_SNODE_H_*/ diff --git a/source/dnode/mgmt/sm/inc/smInt.h b/source/dnode/mgmt/sm/inc/smInt.h index 9290384cab..2c9a43cd98 100644 --- a/source/dnode/mgmt/sm/inc/smInt.h +++ b/source/dnode/mgmt/sm/inc/smInt.h @@ -16,7 +16,7 @@ #ifndef _TD_DND_SNODE_INT_H_ #define _TD_DND_SNODE_INT_H_ -#include "sm.h" +#include "dnd.h" #include "snode.h" #ifdef __cplusplus diff --git a/source/dnode/mgmt/vm/inc/vm.h b/source/dnode/mgmt/vm/inc/vm.h deleted file mode 100644 index 60d9cfc3a1..0000000000 --- a/source/dnode/mgmt/vm/inc/vm.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_VNODES_H_ -#define _TD_DND_VNODES_H_ - -#include "dnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - int32_t openVnodes; - int32_t totalVnodes; - int32_t masterNum; - int64_t numOfSelectReqs; - int64_t numOfInsertReqs; - int64_t numOfInsertSuccessReqs; - int64_t numOfBatchInsertReqs; - int64_t numOfBatchInsertSuccessReqs; -} SVnodesStat; - -void vmGetMgmtFp(SMgmtWrapper *pWrapper); - -void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); -int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); -void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_VNODES_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/vm/inc/vmInt.h b/source/dnode/mgmt/vm/inc/vmInt.h index 6722fe1d65..9f8eac9e97 100644 --- a/source/dnode/mgmt/vm/inc/vmInt.h +++ b/source/dnode/mgmt/vm/inc/vmInt.h @@ -17,7 +17,7 @@ #define _TD_DND_VNODES_INT_H_ #include "sync.h" -#include "vm.h" +#include "dnd.h" #include "vnode.h" #ifdef __cplusplus From 6abfd9fed3eb0d0274cde2f790d06a87fb158041 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:21:20 +0800 Subject: [PATCH 25/42] refact transport --- source/dnode/mgmt/main/inc/dndInt.h | 1 - source/dnode/mgmt/main/src/dndEnv.c | 60 +++++++ source/dnode/mgmt/main/src/dndMsg.c | 80 --------- source/dnode/mgmt/main/src/dndStr.c | 77 --------- source/dnode/mgmt/main/src/dndTransport.c | 199 +++++++++++++--------- 5 files changed, 183 insertions(+), 234 deletions(-) delete mode 100644 source/dnode/mgmt/main/src/dndStr.c diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 5ef52e3942..9477bd06b9 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -37,7 +37,6 @@ void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); // dndMsg.c -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg); // dndFile.c diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/src/dndEnv.c index 9f75594335..8bdc2867d2 100644 --- a/source/dnode/mgmt/main/src/dndEnv.c +++ b/source/dnode/mgmt/main/src/dndEnv.c @@ -57,3 +57,63 @@ void dndCleanup() { taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } + +const char *dndStatStr(EDndStatus status) { + switch (status) { + case DND_STAT_INIT: + return "init"; + case DND_STAT_RUNNING: + return "running"; + case DND_STAT_STOPPED: + return "stopped"; + default: + return "UNKNOWN"; + } +} + +const char *dndNodeLogStr(ENodeType ntype) { + switch (ntype) { + case VNODES: + return "vnode"; + case QNODE: + return "qnode"; + case SNODE: + return "snode"; + case MNODE: + return "mnode"; + case BNODE: + return "bnode"; + default: + return "taosd"; + } +} + +const char *dndNodeProcStr(ENodeType ntype) { + switch (ntype) { + case VNODES: + return "taosv"; + case QNODE: + return "taosq"; + case SNODE: + return "taoss"; + case MNODE: + return "taosm"; + case BNODE: + return "taosb"; + default: + return "taosd"; + } +} + +const char *dndEventStr(EDndEvent ev) { + switch (ev) { + case DND_EVENT_START: + return "start"; + case DND_EVENT_STOP: + return "stop"; + case DND_EVENT_CHILD: + return "child"; + default: + return "UNKNOWN"; + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index 76b66467bf..5d739b7e15 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -16,86 +16,6 @@ #define _DEFAULT_SOURCE #include "dndInt.h" -static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE); - if (pWrapper != NULL) { - dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); - dndReleaseWrapper(pWrapper); - } -} - -static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; - if (msgFp == NULL) { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - - return msgFp; -} - -static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { - SRpcConnInfo connInfo = {0}; - if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { - terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle); - return -1; - } - - memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); - pMsg->clientIp = connInfo.clientIp; - pMsg->clientPort = connInfo.clientPort; - memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); - return 0; -} - -void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { - int32_t code = -1; - SNodeMsg *pMsg = NULL; - NodeMsgFp msgFp = NULL; - - if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { - dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); - } - - if (dndMarkWrapper(pWrapper) != 0) goto _OVER; - if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; - if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; - if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; - - if (pWrapper->procType == PROC_SINGLE) { - dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); - code = (*msgFp)(pWrapper, pMsg); - } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); - } else { - dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, - pMsg->user); - ASSERT(1); - } - -_OVER: - if (code == 0) { - if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is freed in parent process", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - } else { - dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); - if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; - tmsgSendRsp(&rsp); - } - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - - dndReleaseWrapper(pWrapper); -} - static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/src/dndStr.c deleted file mode 100644 index caaa96de59..0000000000 --- a/source/dnode/mgmt/main/src/dndStr.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -const char *dndStatStr(EDndStatus status) { - switch (status) { - case DND_STAT_INIT: - return "init"; - case DND_STAT_RUNNING: - return "running"; - case DND_STAT_STOPPED: - return "stopped"; - default: - return "UNKNOWN"; - } -} - -const char *dndNodeLogStr(ENodeType ntype) { - switch (ntype) { - case VNODES: - return "vnode"; - case QNODE: - return "qnode"; - case SNODE: - return "snode"; - case MNODE: - return "mnode"; - case BNODE: - return "bnode"; - default: - return "taosd"; - } -} - -const char *dndNodeProcStr(ENodeType ntype) { - switch (ntype) { - case VNODES: - return "taosv"; - case QNODE: - return "taosq"; - case SNODE: - return "taoss"; - case MNODE: - return "taosm"; - case BNODE: - return "taosb"; - default: - return "taosd"; - } -} - -const char *dndEventStr(EDndEvent ev) { - switch (ev) { - case DND_EVENT_START: - return "start"; - case DND_EVENT_STOP: - return "stop"; - case DND_EVENT_CHILD: - return "child"; - default: - return "UNKNOWN"; - } -} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index 2be2da4e46..38bc13fec0 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -20,45 +20,133 @@ #define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" -static inline void dndProcessQMVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { - SMsgHead *pHead = pMsg->pCont; - int32_t vgId = htonl(pHead->vgId); - - SMgmtWrapper *pWrapper = pHandle->pWrapper; - if (vgId == QND_VGID) { - pWrapper = pHandle->pQndWrapper; - } else if (vgId == MND_VGID) { - pWrapper = pHandle->pMndWrapper; - } - - dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, - pMsg->handle, pMsg->ahandle, vgId); - dndProcessRpcMsg(pWrapper, pMsg, pEpSet); +static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; + dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet); } -static void dndProcessResponse(SDnode *pDnode, SRpcMsg *pRsp, SEpSet *pEpSet) { - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pRsp->msgType; +static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)]; + if (msgFp == NULL) { + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + } - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); + return msgFp; +} + +static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { + SRpcConnInfo connInfo = {0}; + if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + dError("failed to build msg since %s, app:%p handle:%p", terrstr(), pRpc->ahandle, pRpc->handle); + return -1; + } + + memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); + pMsg->clientIp = connInfo.clientIp; + pMsg->clientPort = connInfo.clientPort; + memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + return 0; +} + +static void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { + int32_t code = -1; + SNodeMsg *pMsg = NULL; + NodeMsgFp msgFp = NULL; + + if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) { + dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet); + } + + if (dndMarkWrapper(pWrapper) != 0) goto _OVER; + if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; + if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; + if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER; + + if (pWrapper->procType == PROC_SINGLE) { + dTrace("msg:%p, is created, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = (*msgFp)(pWrapper, pMsg); + } else if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is created and put into child queue, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); + } else { + dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); + ASSERT(1); + } + +_OVER: + if (code == 0) { + if (pWrapper->procType == PROC_PARENT) { + dTrace("msg:%p, is freed in parent process", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + } else { + dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr()); + if (pRpc->msgType & 1U) { + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + tmsgSendRsp(&rsp); + } + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + dndReleaseWrapper(pWrapper); +} + +static void dndProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { + STransMgmt *pMgmt = &pDnode->trans; + tmsg_t msgType = pMsg->msgType; + bool isReq = msgType & 1u; + SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; + SMgmtWrapper *pWrapper = pHandle->pWrapper; + + if (msgType == TDMT_DND_NETWORK_TEST) { + dTrace("network test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dndProcessStartupReq(pDnode, pMsg); return; } - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), - pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); - dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pRsp, pEpSet); + if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { + dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); } - } else { - dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); - rpcFreeCont(pRsp->pCont); + rpcFreeCont(pMsg->pCont); + return; } + + if (isReq && pMsg->pCont == NULL) { + dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + return; + } + + if (pWrapper == NULL) { + dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); + if (isReq) { + SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rspMsg); + } + rpcFreeCont(pMsg->pCont); + } + + if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { + SMsgHead *pHead = pMsg->pCont; + int32_t vgId = ntohl(pHead->vgId); + if (vgId == QND_VGID) { + pWrapper = pHandle->pQndWrapper; + } else if (vgId == MND_VGID) { + pWrapper = pHandle->pMndWrapper; + } else { + } + } + + dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle); + dndProcessRpcMsg(pWrapper, pMsg, pEpSet); } static int32_t dndInitClient(SDnode *pDnode) { @@ -68,7 +156,7 @@ static int32_t dndInitClient(SDnode *pDnode) { memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.label = "DND"; rpcInit.numOfThreads = 1; - rpcInit.cfp = (RpcCfp)dndProcessResponse; + rpcInit.cfp = (RpcCfp)dndProcessMsg; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = tsShellActivityTimer * 1000; @@ -100,48 +188,6 @@ static void dndCleanupClient(SDnode *pDnode) { } } -static void dndProcessRequest(SDnode *pDnode, SRpcMsg *pReq, SEpSet *pEpSet) { - STransMgmt *pMgmt = &pDnode->trans; - tmsg_t msgType = pReq->msgType; - - if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("network test req will be processed, handle:%p, app:%p", pReq->handle, pReq->ahandle); - dndProcessStartupReq(pDnode, pReq); - return; - } - - if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - dError("req:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - return; - } - - if (pReq->pCont == NULL) { - dTrace("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - return; - } - - SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; - if (pHandle->pWrapper != NULL) { - if (pHandle->pMndWrapper == NULL && pHandle->pQndWrapper == NULL) { - dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, - pReq->handle, pReq->ahandle); - dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); - } else { - dndProcessQMVnodeRpcMsg(pHandle, pReq, pEpSet); - } - } else { - dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; - rpcSendResponse(&rspMsg); - rpcFreeCont(pReq->pCont); - } -} - static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg *pRsp) { SEpSet epSet = {0}; SMgmtWrapper *pWrapper = &pDnode->wrappers[DNODE]; @@ -149,7 +195,8 @@ static inline void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pReq, SRpcMsg rpcSendRecv(pDnode->trans.clientRpc, &epSet, pReq, pRsp); } -static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { +static inline int32_t dndGetHideUserAuth(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, + char *ckey) { int32_t code = 0; char pass[TSDB_PASSWORD_LEN + 1] = {0}; @@ -219,7 +266,7 @@ static int32_t dndInitServer(SDnode *pDnode) { rpcInit.localPort = pDnode->serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = numOfThreads; - rpcInit.cfp = (RpcCfp)dndProcessRequest; + rpcInit.cfp = (RpcCfp)dndProcessMsg; rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; From b2bf8539527bf4c200faf63ca3bcfdf08f091d7b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:22:59 +0800 Subject: [PATCH 26/42] rename file --- source/dnode/mgmt/dm/src/dmMsg.c | 101 ++++++++++++++++++++++++ source/dnode/mgmt/main/src/dndMsg.c | 117 ---------------------------- 2 files changed, 101 insertions(+), 117 deletions(-) delete mode 100644 source/dnode/mgmt/main/src/dndMsg.c diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index 90117044f9..979db7aac0 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -117,6 +117,107 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { return TSDB_CODE_OPS_NOT_SUPPORT; } + +static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper != NULL) { + dndReleaseWrapper(pWrapper); + terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + dError("failed to create node since %s", terrstr()); + return -1; + } + + pWrapper = &pDnode->wrappers[ntype]; + + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); + if (code != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + } else { + dDebug("node:%s, has been opened", pWrapper->name); + pWrapper->deployed = true; + } + + return code; +} + +static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { + SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + dError("failed to drop node since %s", terrstr()); + return -1; + } + + taosWLockLatch(&pWrapper->latch); + pWrapper->deployed = false; + + int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); + if (code != 0) { + pWrapper->deployed = true; + dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); + } else { + pWrapper->deployed = false; + dDebug("node:%s, has been dropped", pWrapper->name); + } + + taosWUnLockLatch(&pWrapper->latch); + dndReleaseWrapper(pWrapper); + return code; +} + +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { + switch (pMsg->rpcMsg.msgType) { + case TDMT_DND_CREATE_MNODE: + return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_DROP_MNODE: + return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + case TDMT_DND_CREATE_QNODE: + return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_DROP_QNODE: + return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + case TDMT_DND_CREATE_SNODE: + return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_DROP_SNODE: + return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + case TDMT_DND_CREATE_BNODE: + return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + case TDMT_DND_DROP_BNODE: + return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + default: + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } +} + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} + +void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { + memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); + pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); +} + +void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { + dDebug("startup req is received"); + SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); + dndGetStartup(pDnode, pStartup); + + dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); + SRpcMsg rpcRsp = { + .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; + rpcSendResponse(&rpcRsp); +} + void dmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID); diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c deleted file mode 100644 index 5d739b7e15..0000000000 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dndInt.h" - -static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper != NULL) { - dndReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; - dError("failed to create node since %s", terrstr()); - return -1; - } - - pWrapper = &pDnode->wrappers[ntype]; - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); - return -1; - } - - int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg); - if (code != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); - } else { - dDebug("node:%s, has been opened", pWrapper->name); - pWrapper->deployed = true; - } - - return code; -} - -static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { - SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); - if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; - dError("failed to drop node since %s", terrstr()); - return -1; - } - - taosWLockLatch(&pWrapper->latch); - pWrapper->deployed = false; - - int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg); - if (code != 0) { - pWrapper->deployed = true; - dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); - } else { - pWrapper->deployed = false; - dDebug("node:%s, has been dropped", pWrapper->name); - } - - taosWUnLockLatch(&pWrapper->latch); - dndReleaseWrapper(pWrapper); - return code; -} - -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { - switch (pMsg->rpcMsg.msgType) { - case TDMT_DND_CREATE_MNODE: - return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_DROP_MNODE: - return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); - case TDMT_DND_CREATE_QNODE: - return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_DROP_QNODE: - return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); - case TDMT_DND_CREATE_SNODE: - return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_DROP_SNODE: - return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); - case TDMT_DND_CREATE_BNODE: - return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); - case TDMT_DND_DROP_BNODE: - return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); - default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - return -1; - } -} - -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { - memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); - pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); -} - -void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { - dDebug("startup req is received"); - SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq)); - dndGetStartup(pDnode, pStartup); - - dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished); - SRpcMsg rpcRsp = { - .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle}; - rpcSendResponse(&rpcRsp); -} From a554bd7fe7ceb370a3d9e4deaf6d243434eeb94b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:24:20 +0800 Subject: [PATCH 27/42] rename file --- source/dnode/mgmt/bm/src/{bmMsg.c => bmHandle.c} | 0 source/dnode/mgmt/dm/src/{dmMsg.c => dmHandle.c} | 0 source/dnode/mgmt/mm/src/{mmMsg.c => mmHandle.c} | 0 source/dnode/mgmt/qm/src/{qmMsg.c => qmHandle.c} | 0 source/dnode/mgmt/sm/src/{smMsg.c => smHandle.c} | 0 source/dnode/mgmt/vm/src/{vmMsg.c => vmHandle.c} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename source/dnode/mgmt/bm/src/{bmMsg.c => bmHandle.c} (100%) rename source/dnode/mgmt/dm/src/{dmMsg.c => dmHandle.c} (100%) rename source/dnode/mgmt/mm/src/{mmMsg.c => mmHandle.c} (100%) rename source/dnode/mgmt/qm/src/{qmMsg.c => qmHandle.c} (100%) rename source/dnode/mgmt/sm/src/{smMsg.c => smHandle.c} (100%) rename source/dnode/mgmt/vm/src/{vmMsg.c => vmHandle.c} (100%) diff --git a/source/dnode/mgmt/bm/src/bmMsg.c b/source/dnode/mgmt/bm/src/bmHandle.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmMsg.c rename to source/dnode/mgmt/bm/src/bmHandle.c diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmHandle.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmMsg.c rename to source/dnode/mgmt/dm/src/dmHandle.c diff --git a/source/dnode/mgmt/mm/src/mmMsg.c b/source/dnode/mgmt/mm/src/mmHandle.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmMsg.c rename to source/dnode/mgmt/mm/src/mmHandle.c diff --git a/source/dnode/mgmt/qm/src/qmMsg.c b/source/dnode/mgmt/qm/src/qmHandle.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmMsg.c rename to source/dnode/mgmt/qm/src/qmHandle.c diff --git a/source/dnode/mgmt/sm/src/smMsg.c b/source/dnode/mgmt/sm/src/smHandle.c similarity index 100% rename from source/dnode/mgmt/sm/src/smMsg.c rename to source/dnode/mgmt/sm/src/smHandle.c diff --git a/source/dnode/mgmt/vm/src/vmMsg.c b/source/dnode/mgmt/vm/src/vmHandle.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmMsg.c rename to source/dnode/mgmt/vm/src/vmHandle.c From 7e361ebcd09a9064fed94b65a3bfdff34f30bf8f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 19:52:04 +0800 Subject: [PATCH 28/42] refact dm --- include/util/tdef.h | 6 +- source/dnode/mgmt/dm/inc/dmInt.h | 7 +- source/dnode/mgmt/dm/src/dmHandle.c | 37 ++--- source/dnode/mgmt/dm/src/dmInt.c | 2 +- source/dnode/mgmt/dm/src/dmWorker.c | 53 +++---- source/dnode/mgmt/main/src/dndInt.c | 7 + source/dnode/mgmt/main/src/dndTransport.c | 8 +- source/dnode/mgmt/mm/src/mmHandle.c | 162 ++++++++++----------- source/dnode/mgmt/qm/src/qmHandle.c | 18 +-- source/dnode/mgmt/sm/src/smHandle.c | 4 +- source/dnode/mgmt/vm/src/vmHandle.c | 88 +++++------ source/dnode/mnode/impl/src/mndQuery.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 2 +- 13 files changed, 200 insertions(+), 196 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index be88afcf66..f1af9eca1f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -480,9 +480,9 @@ enum { SND_WORKER_TYPE__UNIQUE, }; -#define MND_VGID -1 -#define QND_VGID 1 -#define VND_VGID 0 +#define MNODE_HANDLE -1 +#define QNODE_HANDLE 1 +#define DEFAULT_HANDLE 0 #define MAX_NUM_STR_SIZE 40 diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index a8f46ef7e2..98abb8ef54 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -45,7 +45,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); // dmMsg.c -void dmInitMsgHandles(SMgmtWrapper *pWrapper); +void dmInitMsgHandle(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); @@ -53,10 +53,11 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); // dmWorker.c +int32_t dmStartThread(SDnodeMgmt *pMgmt); int32_t dmStartWorker(SDnodeMgmt *pMgmt); void dmStopWorker(SDnodeMgmt *pMgmt); -int32_t dmStartThread(SDnodeMgmt *pMgmt); -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmPutMsgToMgmtWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); +int32_t dmPutMsgToStatusWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/dm/src/dmHandle.c b/source/dnode/mgmt/dm/src/dmHandle.c index 979db7aac0..07eca3f230 100644 --- a/source/dnode/mgmt/dm/src/dmHandle.c +++ b/source/dnode/mgmt/dm/src/dmHandle.c @@ -195,14 +195,7 @@ int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { } } -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { - SStartupReq *pStartup = &pDnode->startup; - tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); - tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); - pStartup->finished = 0; -} - -void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { +static void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq)); pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING); } @@ -218,21 +211,21 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { rpcSendResponse(&rpcRsp); } -void dmInitMsgHandles(SMgmtWrapper *pWrapper) { +void dmInitMsgHandle(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmPutMsgToStatusWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmPutMsgToMgmtWorker, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index 8c46c402ab..1e38b8304d 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -165,7 +165,7 @@ void dmGetMgmtFp(SMgmtWrapper *pWrapper) { mgmtFp.startFp = dmStart; mgmtFp.requiredFp = dmRequire; - dmInitMsgHandles(pWrapper); + dmInitMsgHandle(pWrapper); pWrapper->name = "dnode"; pWrapper->fp = mgmtFp; } diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/src/dmWorker.c index 9045a44232..6061189ca1 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/src/dmWorker.c @@ -18,7 +18,7 @@ static void *dmThreadRoutine(void *param) { SDnodeMgmt *pMgmt = param; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; int64_t lastStatusTime = taosGetTimestampMs(); int64_t lastMonitorTime = lastStatusTime; @@ -32,8 +32,7 @@ static void *dmThreadRoutine(void *param) { } int64_t curTime = taosGetTimestampMs(); - - float statusInterval = (curTime - lastStatusTime) / 1000.0f; + float statusInterval = (curTime - lastStatusTime) / 1000.0f; if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) { dmSendStatusReq(pMgmt); lastStatusTime = curTime; @@ -47,10 +46,21 @@ static void *dmThreadRoutine(void *param) { } } +int32_t dmStartThread(SDnodeMgmt *pMgmt) { + pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); + if (pMgmt->threadId == NULL) { + dError("failed to init dnode thread"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pInfo->ahandle; - SDnode * pDnode = pMgmt->pDnode; + SDnode *pDnode = pMgmt->pDnode; SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; dTrace("msg:%p, will be processed in dnode queue", pMsg); @@ -95,16 +105,14 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { } int32_t dmStartWorker(SDnodeMgmt *pMgmt) { - SSingleWorkerCfg mgmtCfg = { - .min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) { + SSingleWorkerCfg mcfg = {.min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mcfg) != 0) { dError("failed to start dnode mgmt worker since %s", terrstr()); return -1; } - SSingleWorkerCfg statusCfg = { - .min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; - if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) { + SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt}; + if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) { dError("failed to start dnode status worker since %s", terrstr()); return -1; } @@ -113,17 +121,6 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) { return 0; } -int32_t dmStartThread(SDnodeMgmt *pMgmt) { - pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt); - if (pMgmt->threadId == NULL) { - dError("failed to init dnode thread"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - void dmStopWorker(SDnodeMgmt *pMgmt) { tSingleWorkerCleanup(&pMgmt->mgmtWorker); tSingleWorkerCleanup(&pMgmt->statusWorker); @@ -135,12 +132,18 @@ void dmStopWorker(SDnodeMgmt *pMgmt) { dDebug("dnode workers are closed"); } -int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { +int32_t dmPutMsgToMgmtWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; - if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) { - pWorker = &pMgmt->statusWorker; - } + + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + taosWriteQitem(pWorker->queue, pMsg); + return 0; +} + +int32_t dmPutMsgToStatusWorker(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { + SDnodeMgmt *pMgmt = pWrapper->pMgmt; + SSingleWorker *pWorker = &pMgmt->statusWorker; dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 5e4b0a1f7b..72feed7c62 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -204,3 +204,10 @@ void dndSetStatus(SDnode *pDnode, EDndStatus status) { pDnode->status = status; } } + +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) { + SStartupReq *pStartup = &pDnode->startup; + tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN); + tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN); + pStartup->finished = 0; +} diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/src/dndTransport.c index 38bc13fec0..babc6d1209 100644 --- a/source/dnode/mgmt/main/src/dndTransport.c +++ b/source/dnode/mgmt/main/src/dndTransport.c @@ -137,9 +137,9 @@ static void dndProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { if (pHandle->pMndWrapper != NULL || pHandle->pQndWrapper != NULL) { SMsgHead *pHead = pMsg->pCont; int32_t vgId = ntohl(pHead->vgId); - if (vgId == QND_VGID) { + if (vgId == QNODE_HANDLE) { pWrapper = pHandle->pQndWrapper; - } else if (vgId == MND_VGID) { + } else if (vgId == MNODE_HANDLE) { pWrapper = pHandle->pMndWrapper; } else { } @@ -315,13 +315,13 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { if (msgFp == NULL) continue; SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; - if (vgId == QND_VGID) { + if (vgId == QNODE_HANDLE) { if (pHandle->pQndWrapper != NULL) { dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); return -1; } pHandle->pQndWrapper = pWrapper; - } else if (vgId == MND_VGID) { + } else if (vgId == MNODE_HANDLE) { if (pHandle->pMndWrapper != NULL) { dError("msg:%s has multiple process nodes", tMsgInfo[msgIndex]); return -1; diff --git a/source/dnode/mgmt/mm/src/mmHandle.c b/source/dnode/mgmt/mm/src/mmHandle.c index 6afcd249b3..666f9daa91 100644 --- a/source/dnode/mgmt/mm/src/mmHandle.c +++ b/source/dnode/mgmt/mm/src/mmHandle.c @@ -75,91 +75,91 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE); } diff --git a/source/dnode/mgmt/qm/src/qmHandle.c b/source/dnode/mgmt/qm/src/qmHandle.c index da5ba6472a..14ac08af05 100644 --- a/source/dnode/mgmt/qm/src/qmHandle.c +++ b/source/dnode/mgmt/qm/src/qmHandle.c @@ -56,14 +56,14 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void qmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QNODE_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QNODE_HANDLE); } diff --git a/source/dnode/mgmt/sm/src/smHandle.c b/source/dnode/mgmt/sm/src/smHandle.c index c522ef7fc3..dc5ee1155a 100644 --- a/source/dnode/mgmt/sm/src/smHandle.c +++ b/source/dnode/mgmt/sm/src/smHandle.c @@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void smInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by SNODE - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mgmt/vm/src/vmHandle.c b/source/dnode/mgmt/vm/src/vmHandle.c index c3ad74d246..6a51c9a77f 100644 --- a/source/dnode/mgmt/vm/src/vmHandle.c +++ b/source/dnode/mgmt/vm/src/vmHandle.c @@ -235,49 +235,49 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { void vmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, VND_VGID); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, VND_VGID); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 40b4f60bd4..d184e354c4 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -52,7 +52,7 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) { } int32_t mndInitQuery(SMnode *pMnode) { - if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { + if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { mError("failed to init qworker in mnode since %s", terrstr()); return -1; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 11baeff04a..5e466c8cda 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -419,7 +419,7 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - SQueryNodeAddr addr = { .nodeId = MND_VGID, .epSet = pCxt->pPlanCxt->mgmtEpSet }; + SQueryNodeAddr addr = { .nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet }; taosArrayPush(pCxt->pExecNodeList, &addr); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; From 469cd23f43c38f3dbf9112eb557eceb4181380dd Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Apr 2022 07:57:44 -0400 Subject: [PATCH 29/42] integrate constant calculate --- include/libs/nodes/nodes.h | 16 +-- include/libs/nodes/querynodes.h | 1 + source/libs/executor/src/executorimpl.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 25 +--- source/libs/nodes/src/nodesCodeFuncs.c | 6 + source/libs/nodes/src/nodesTraverseFuncs.c | 48 ++++---- source/libs/nodes/test/nodesTest.cpp | 2 +- source/libs/parser/CMakeLists.txt | 2 +- source/libs/parser/inc/parInt.h | 5 +- source/libs/parser/src/parAstCreater.c | 36 +++--- source/libs/parser/src/parAstParser.c | 2 +- source/libs/parser/src/parCalcConst.c | 128 +++++++++++++++++++++ source/libs/parser/src/parTranslater.c | 31 ++--- source/libs/parser/src/parser.c | 7 +- source/libs/parser/test/parserAstTest.cpp | 4 +- source/libs/planner/src/planLogicCreater.c | 4 +- source/libs/planner/src/planPhysiCreater.c | 6 +- source/libs/planner/test/plannerTest.cpp | 7 ++ source/libs/scalar/src/filter.c | 8 +- source/libs/scalar/src/scalar.c | 4 +- 20 files changed, 234 insertions(+), 110 deletions(-) create mode 100644 source/libs/parser/src/parCalcConst.c diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 9668082696..d2217f7e01 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -213,16 +213,16 @@ typedef enum EDealRes { } EDealRes; typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext); -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext); -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext); +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext); +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext); typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext); -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext); -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext); +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext); bool nodesEqualNode(const SNodeptr a, const SNodeptr b); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index ca4646a5ca..a8ff48072f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -79,6 +79,7 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; + bool genByCalc; union { bool b; int64_t i; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 169e47eb33..a0555ab8cd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -5801,7 +5801,7 @@ void getDBNameFromCondition(SNode* pCondition, char* dbName) { return; } - nodesWalkNode(pCondition, getDBNameFromConditionWalker, dbName); + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, dbName); } static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index f8bc99b975..ac78988900 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -113,36 +113,13 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { } static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { + COPY_ALL_SCALAR_FIELDS; exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst); COPY_CHAR_POINT_FIELD(literal); - COPY_SCALAR_FIELD(isDuration); - COPY_SCALAR_FIELD(translate); if (!pSrc->translate) { return (SNode*)pDst; } switch (pSrc->node.resType.type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - COPY_SCALAR_FIELD(datum.b); - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - COPY_SCALAR_FIELD(datum.i); - break; - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - COPY_SCALAR_FIELD(datum.u); - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - COPY_SCALAR_FIELD(datum.d); - break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 5337965164..e39a7ecfa6 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1544,6 +1544,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkValueTranslate, pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -1614,6 +1617,9 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->genByCalc); + } + if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index bbd0473edd..15c078d8d2 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -138,19 +138,19 @@ static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalke return DEAL_RES_CONTINUE; } -void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkList(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { +void nodesWalkExprs(SNodeList* pNodeList, FNodeWalker walker, void* pContext) { (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); } -void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { +void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext) { (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); } -void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { +void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext) { (void)walkList(pList, TRAVERSAL_POSTORDER, walker, pContext); } @@ -267,19 +267,19 @@ static EDealRes rewriteList(SNodeList* pNodeList, ETraversalOrder order, FNodeRe return DEAL_RES_CONTINUE; } -void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_PREORDER, rewriter, pContext); } -void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext) { (void)rewriteNode(pNode, TRAVERSAL_POSTORDER, rewriter, pContext); } -void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { +void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext) { (void)rewriteList(pList, TRAVERSAL_POSTORDER, rewriter, pContext); } @@ -290,20 +290,20 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa switch (clause) { case SQL_CLAUSE_FROM: - nodesWalkNode(pSelect->pFromTable, walker, pContext); - nodesWalkNode(pSelect->pWhere, walker, pContext); + nodesWalkExpr(pSelect->pFromTable, walker, pContext); + nodesWalkExpr(pSelect->pWhere, walker, pContext); case SQL_CLAUSE_WHERE: - nodesWalkList(pSelect->pPartitionByList, walker, pContext); + nodesWalkExprs(pSelect->pPartitionByList, walker, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesWalkNode(pSelect->pWindow, walker, pContext); + nodesWalkExpr(pSelect->pWindow, walker, pContext); case SQL_CLAUSE_WINDOW: - nodesWalkList(pSelect->pGroupByList, walker, pContext); + nodesWalkExprs(pSelect->pGroupByList, walker, pContext); case SQL_CLAUSE_GROUP_BY: - nodesWalkNode(pSelect->pHaving, walker, pContext); + nodesWalkExpr(pSelect->pHaving, walker, pContext); case SQL_CLAUSE_HAVING: - nodesWalkList(pSelect->pOrderByList, walker, pContext); + nodesWalkExprs(pSelect->pOrderByList, walker, pContext); case SQL_CLAUSE_ORDER_BY: - nodesWalkList(pSelect->pProjectionList, walker, pContext); + nodesWalkExprs(pSelect->pProjectionList, walker, pContext); case SQL_CLAUSE_SELECT: default: break; @@ -319,20 +319,20 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit switch (clause) { case SQL_CLAUSE_FROM: - nodesRewriteNode(&(pSelect->pFromTable), rewriter, pContext); - nodesRewriteNode(&(pSelect->pWhere), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pFromTable), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWhere), rewriter, pContext); case SQL_CLAUSE_WHERE: - nodesRewriteList(pSelect->pPartitionByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pPartitionByList, rewriter, pContext); case SQL_CLAUSE_PARTITION_BY: - nodesRewriteNode(&(pSelect->pWindow), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pWindow), rewriter, pContext); case SQL_CLAUSE_WINDOW: - nodesRewriteList(pSelect->pGroupByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pGroupByList, rewriter, pContext); case SQL_CLAUSE_GROUP_BY: - nodesRewriteNode(&(pSelect->pHaving), rewriter, pContext); + nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext); case SQL_CLAUSE_HAVING: - nodesRewriteList(pSelect->pProjectionList, rewriter, pContext); + nodesRewriteExprs(pSelect->pProjectionList, rewriter, pContext); case SQL_CLAUSE_SELECT: - nodesRewriteList(pSelect->pOrderByList, rewriter, pContext); + nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext); default: break; } diff --git a/source/libs/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp index fd4a9e8c20..fdfb9522db 100644 --- a/source/libs/nodes/test/nodesTest.cpp +++ b/source/libs/nodes/test/nodesTest.cpp @@ -49,7 +49,7 @@ TEST(NodesTest, traverseTest) { EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); EDealRes res = DEAL_RES_CONTINUE; - nodesRewriteNodePostOrder(&pRoot, rewriterTest, &res); + nodesRewriteExprPostOrder(&pRoot, rewriterTest, &res); EXPECT_EQ(res, DEAL_RES_CONTINUE); EXPECT_EQ(nodeType(pRoot), QUERY_NODE_VALUE); EXPECT_EQ(string(((SValueNode*)pRoot)->literal), "18"); diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 5d02868657..c3157480a9 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util nodes catalog function transport qcom + PRIVATE os util nodes catalog function scalar transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 9bad3e9eb9..ea01caf9d7 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -23,9 +23,10 @@ extern "C" { #include "parser.h" int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery); -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery); +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 18a0ae8cfa..07316ddd39 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -243,27 +243,27 @@ static SDatabaseOptions* setDbRetentions(SAstCreateContext* pCxt, SDatabaseOptio return pOptions; } - char val[20] = {0}; - int32_t len = trimString(pVal->z, pVal->n, val, sizeof(val)); - char* pStart = val; - char* pEnd = val + len; - int32_t sepOrder = 1; - while (1) { - char* pPos = strchr(pStart, (0 == sepOrder % 2) ? ',' : ':'); - SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { - pCxt->valid = false; - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); - return pOptions; + if (pVal->n > 2) { + char* pStart = pVal->z + 1; + char* pEnd = pVal->z + pVal->n - 1; + int32_t sepOrder = 1; + while (1) { + char* pPos = strchr(pStart, (0 == (sepOrder++) % 2) ? ',' : ':'); + SToken t = { .type = TK_NK_VARIABLE, .z = pStart, .n = (NULL == pPos ? pEnd - pStart : pPos - pStart)}; + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pOptions->pRetentions, createDurationValueNode(pCxt, &t))) { + pCxt->valid = false; + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "Out of memory"); + return pOptions; + } + if (NULL == pPos) { + break; + } + pStart = pPos + 1; } - if (NULL == pPos) { - break; - } - pStart = pPos + 1; } - if (LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", val); + if (LIST_LENGTH(pOptions->pRetentions) < 2 || LIST_LENGTH(pOptions->pRetentions) % 2 != 0) { + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid db option retentions: %s", pVal->z); pCxt->valid = false; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 5c38ccaff8..76af6b7ac7 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -26,7 +26,7 @@ extern void Parse(void*, int, SToken, void*); extern void ParseFree(void*, FFree); extern void ParseTrace(FILE*, char*); -int32_t doParse(SParseContext* pParseCxt, SQuery** pQuery) { +int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { SAstCreateContext cxt; initAstCreateContext(pParseCxt, &cxt); void *pParser = ParseAlloc((FMalloc)taosMemoryMalloc); diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c new file mode 100644 index 0000000000..a572ccf5bd --- /dev/null +++ b/source/libs/parser/src/parCalcConst.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parInt.h" +#include "scalar.h" + +typedef struct SCalcConstContext { + int32_t code; +} SCalcConstContext; + +static int32_t calcConstQuery(SNode* pStmt); + +static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { + SNode* pNew = NULL; + pCxt->code = scalarCalculateConstants(*pNode, &pNew); + if (TSDB_CODE_SUCCESS != pCxt->code) { + return DEAL_RES_ERROR; + } + ((SValueNode*)pNew)->genByCalc = true; + ((SValueNode*)pNew)->translate = true; + *pNode = pNew; + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { + SOperatorNode* pOp = *pNode; + if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); + } + return DEAL_RES_CONTINUE; +} + +static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { + SFunctionNode* pFunc = *pNode; + SNode* pParam = NULL; + FOREACH(pParam, pFunc->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) { + SLogicConditionNode* pCond = *pNode; + SNode* pParam = NULL; + FOREACH(pParam, pCond->pParameterList) { + if (QUERY_NODE_VALUE != nodeType(pParam)) { + return DEAL_RES_CONTINUE; + } + } + return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); +} + +static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) { + SCalcConstContext* pCxt = pContext; + pCxt->code = calcConstQuery((*pNode)->pSubquery); + return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + +static EDealRes calcConst(SNode** pNode, void* pContext) { + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + return calcConstOperator((SOperatorNode**)pNode, pContext); + case QUERY_NODE_FUNCTION: + return calcConstFunction((SFunctionNode**)pNode, pContext); + case QUERY_NODE_LOGIC_CONDITION: + return calcConstLogicCond((SLogicConditionNode**)pNode, pContext); + case QUERY_NODE_TEMP_TABLE: + return calcConstSubquery((STempTableNode**)pNode, pContext); + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t calcConstSelect(SSelectStmt* pSelect) { + SCalcConstContext cxt = { .code = TSDB_CODE_SUCCESS }; + nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, &cxt); + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWhere, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprPostOrder(&pSelect->pHaving, calcConst, &cxt); + } + if (TSDB_CODE_SUCCESS == cxt.code) { + nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, &cxt); + } + return cxt.code; +} + +static int32_t calcConstQuery(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return calcConstSelect((SSelectStmt*)pStmt); + default: + break; + } + return TSDB_CODE_SUCCESS; +} + +int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { + return calcConstQuery(pQuery->pRoot); +} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 641750cd98..850ef9a4ac 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -50,14 +50,12 @@ static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -static EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t errCode, ...) { - va_list vArgList; - va_start(vArgList, errCode); - generateSyntaxErrMsg(&pCxt->msgBuf, errCode, vArgList); - va_end(vArgList); - pCxt->errCode = errCode; - return DEAL_RES_ERROR; -} +#define generateDealNodeErrMsg(pCxt, code, ...) \ + ({ \ + generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ + pCxt->errCode = code; \ + DEAL_RES_ERROR; \ + }) static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); @@ -440,6 +438,9 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if (nodesIsUnaryOp(pOp)) { + if (OP_TYPE_MINUS == pOp->opType && !IS_NUMERIC_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); + } return DEAL_RES_CONTINUE; } SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; @@ -510,12 +511,12 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { } static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + nodesWalkExprPostOrder(pNode, doTranslateExpr, pCxt); return pCxt->errCode; } static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { - nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + nodesWalkExprsPostOrder(pList, doTranslateExpr, pCxt); return pCxt->errCode; } @@ -570,7 +571,7 @@ static EDealRes doCheckExprForGroupBy(SNode* pNode, void* pContext) { } static int32_t checkExprForGroupBy(STranslateContext* pCxt, SNode* pNode) { - nodesWalkNode(pNode, doCheckExprForGroupBy, pCxt); + nodesWalkExpr(pNode, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -578,7 +579,7 @@ static int32_t checkExprListForGroupBy(STranslateContext* pCxt, SNodeList* pList if (NULL == getGroupByList(pCxt)) { return TSDB_CODE_SUCCESS; } - nodesWalkList(pList, doCheckExprForGroupBy, pCxt); + nodesWalkExprs(pList, doCheckExprForGroupBy, pCxt); return pCxt->errCode; } @@ -605,9 +606,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) return TSDB_CODE_SUCCESS; } CheckAggColCoexistCxt cxt = { .pTranslateCxt = pCxt, .existAggFunc = false, .existCol = false }; - nodesWalkList(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkList(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (cxt.existAggFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_SINGLE_GROUP); @@ -2633,7 +2634,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { +int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = { .pParseCxt = pParseCxt, .errCode = TSDB_CODE_SUCCESS, diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d410aa2e17..ebe76cc129 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -30,9 +30,12 @@ static bool isInsertSql(const char* pStr, size_t length) { } static int32_t parseSqlIntoAst(SParseContext* pCxt, SQuery** pQuery) { - int32_t code = doParse(pCxt, pQuery); + int32_t code = parse(pCxt, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = doTranslate(pCxt, *pQuery); + code = translate(pCxt, *pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = calculateConstant(pCxt, *pQuery); } return code; } diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 6dc3b53ae3..72670bb79a 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -54,7 +54,7 @@ private: static const int max_err_len = 1024; bool runImpl(int32_t parseCode, int32_t translateCode) { - int32_t code = doParse(&cxt_, &query_); + int32_t code = parse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { parseErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == parseCode); @@ -63,7 +63,7 @@ private: return false; } parsedAstStr_ = toString(query_->pRoot); - code = doTranslate(&cxt_, query_); + code = translate(&cxt_, query_); if (code != TSDB_CODE_SUCCESS) { translateErrStr_ = string("code:") + tstrerror(code) + string(", msg:") + errMagBuf_; return (terrno == translateCode); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index f57f476d51..149071bad2 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -83,7 +83,7 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { } static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { - nodesWalkList(pExprs, doNameExpr, NULL); + nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; @@ -384,7 +384,7 @@ static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pEx return TSDB_CODE_OUT_OF_MEMORY; } - nodesWalkList(pExprs, doCreateColumn, &cxt); + nodesWalkExprs(pExprs, doCreateColumn, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(cxt.pList); return cxt.errCode; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 11baeff04a..4e729a051f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -264,7 +264,7 @@ static int32_t setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkNode(pRes, doSetSlotId, &cxt); + nodesWalkExpr(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return cxt.errCode; @@ -285,7 +285,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; - nodesWalkList(pRes, doSetSlotId, &cxt); + nodesWalkExprs(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return cxt.errCode; @@ -606,7 +606,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN } } SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; - nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { nodesDestroyList(cxt.pPrecalcExprs); *pPrecalcExprs = NULL; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index ad76c8f879..32b221c4fb 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -157,6 +157,13 @@ TEST_F(PlannerTest, simple) { ASSERT_TRUE(run()); } +TEST_F(PlannerTest, selectConstant) { + setDatabase("root", "test"); + + bind("SELECT 2-1 FROM t1"); + ASSERT_TRUE(run()); +} + TEST_F(PlannerTest, stSimple) { setDatabase("root", "test"); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b0632bbc34..23a22dfabf 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1297,7 +1297,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { resGroup = taosArrayInit(4, sizeof(SFilterGroup)); SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); @@ -1322,7 +1322,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { if (LOGIC_COND_TYPE_OR == node->condType) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { - nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + nodesWalkExpr(cell->pNode, fltTreeToGroup, (void *)pContext); FLT_ERR_JRET(ctx->code); cell = cell->pNext; @@ -3190,7 +3190,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { filterInitUnitsFields(info); SFltBuildGroupCtx tctx = {.info = info, .group = group}; - nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); @@ -3566,7 +3566,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + nodesRewriteExprPostOrder(pNode, fltReviseRewriter, (void *)pStat); FLT_RET(pStat->code); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 1cc259d4da..08ad7bfa29 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -673,7 +673,7 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); + nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); SCL_ERR_JRET(ctx.code); *pRes = pNode; @@ -696,7 +696,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); + nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); SCL_ERR_JRET(ctx.code); if (pDst) { From 8e2d121b3ad4ce4e5f73907f1543fef1e0557586 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 19:59:25 +0800 Subject: [PATCH 30/42] temp --- source/client/src/tmq.c | 55 ++++++++++++++-------- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- source/util/src/tlog.c | 8 ++-- 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 426a62433b..81219e1c36 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -79,6 +79,7 @@ struct tmq_t { tmq_commit_cb* commit_cb; int32_t nextTopicIdx; int8_t epStatus; + int32_t epSkipCnt; int32_t waitingRequest; int32_t readyRequest; SArray* clientTopics; // SArray @@ -107,6 +108,7 @@ typedef struct { // connection info int32_t vgId; int32_t vgStatus; + int64_t skipCnt; SEpSet epSet; } SMqClientVg; @@ -313,6 +315,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->waitingRequest = 0; pTmq->readyRequest = 0; pTmq->epStatus = 0; + pTmq->epSkipCnt = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -348,6 +351,8 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->epoch = 0; pTmq->waitingRequest = 0; pTmq->readyRequest = 0; + pTmq->epStatus = 0; + pTmq->epSkipCnt = 0; // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -842,7 +847,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); return 0; } @@ -892,26 +897,26 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { } #endif - tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset, + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset, pRsp->msg.rspOffset); pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return 0; CREATE_MSG_FAIL: if (pParam->epoch == tmq->epoch) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return code; } bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ - tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch); + tscDebug("consumer %ld update ep epoch %d to epoch %d", tmq->consumerId, tmq->epoch, epoch); bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; @@ -942,7 +947,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d build %s\n", tmq->consumerId, epoch, pVgCur->vgId, vgKey); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -956,18 +961,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset og to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d found %s\n", tmq->consumerId, epoch, pVgEp->vgId, vgKey); } - tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, + .skipCnt = 0, }; taosArrayPush(topic.vgs, &clientVg); set = true; @@ -984,9 +990,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; - tscDebug("consumer %ld recv ep", tmq->consumerId); if (code != 0) { - tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync); + tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); goto END; } @@ -995,6 +1000,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { // Epoch will only increase when received newer epoch ep msg SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); + tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); if (head->epoch <= epoch) { goto END; } @@ -1019,7 +1025,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp); taosWriteQitem(tmq->mqueue, pRsp); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ } END: @@ -1033,9 +1039,11 @@ END: int32_t tmqAskEp(tmq_t* tmq, bool sync) { int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); if (epStatus == 1) { - tscDebug("consumer %ld skip ask ep", tmq->consumerId); - return 0; + int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); + tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); + if (epSkipCnt < 40) return 0; } + atomic_store_32(&tmq->epSkipCnt, 0); int32_t tlen = sizeof(SMqCMGetSubEpReq); SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); if (req == NULL) { @@ -1221,20 +1229,29 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { - tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId); + int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1); + tscDebug("consumer %ld skip vg %d skip cnt %ld", tmq->consumerId, pVg->vgId, skipCnt); continue; +#if 0 + if (skipCnt < 30000) { + continue; + } else { + tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId); + } +#endif } + atomic_store_64(&pVg->skipCnt, 0); SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); if (pReq == NULL) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { taosMemoryFree(pReq); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } pParam->tmq = tmq; @@ -1247,7 +1264,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { taosMemoryFree(pReq); taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - tsem_post(&tmq->rspSem); + /*tsem_post(&tmq->rspSem);*/ return -1; } @@ -1379,7 +1396,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmqAskEp(tmq, false); tmqPollImpl(tmq, blocking_time); - tsem_wait(&tmq->rspSem); + /*tsem_wait(&tmq->rspSem);*/ rspMsg = tmqHandleAllRsp(tmq, blocking_time, false); if (rspMsg) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 211163ce35..ae9a4198a4 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -216,7 +216,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { // TODO int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mTrace("try to get sub ep, old val: %d", hbStatus); + mTrace("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, pConsumer->epoch, hbStatus); atomic_store_32(&pConsumer->hbStatus, 0); /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index ef15f44f8f..f6fec47196 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -214,6 +214,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; + taosSsleep(2); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); @@ -347,12 +348,13 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { taosThreadMutexInit(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); + TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->pFile == NULL) { + if (pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } + tsLogObj.logHandle->pFile = pFile; taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines @@ -746,4 +748,4 @@ void taosSetAllDebugFlag(int32_t flag) { fsDebugFlag = flag; uInfo("all debug flag are set to %d", flag); -} \ No newline at end of file +} From e00f74ccbc3f7f1b33e91c5a0ace429120d7b1b4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 20:05:42 +0800 Subject: [PATCH 31/42] refact dm --- source/dnode/mgmt/bm/src/bmInt.c | 2 +- source/dnode/mgmt/dm/inc/dmInt.h | 9 +++- source/dnode/mgmt/dm/src/dmHandle.c | 22 +++++----- source/dnode/mgmt/dm/src/dmInt.c | 8 ++-- source/dnode/mgmt/dm/src/dmWorker.c | 14 +----- source/dnode/mgmt/main/inc/dnd.h | 67 ++++++++++++++++++++--------- source/dnode/mgmt/main/src/dndInt.c | 12 +++--- source/dnode/mgmt/mm/src/mmInt.c | 2 +- source/dnode/mgmt/qm/src/qmInt.c | 2 +- source/dnode/mgmt/sm/src/smInt.c | 2 +- source/dnode/mgmt/vm/src/vmInt.c | 2 +- 11 files changed, 83 insertions(+), 59 deletions(-) diff --git a/source/dnode/mgmt/bm/src/bmInt.c b/source/dnode/mgmt/bm/src/bmInt.c index 4ca7afbb6d..2cb0f50dfe 100644 --- a/source/dnode/mgmt/bm/src/bmInt.c +++ b/source/dnode/mgmt/bm/src/bmInt.c @@ -109,7 +109,7 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) { return code; } -void bmGetMgmtFp(SMgmtWrapper *pWrapper) { +void bmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = bmOpen; mgmtFp.closeFp = bmClose; diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/dm/inc/dmInt.h index 98abb8ef54..e68f7ddb65 100644 --- a/source/dnode/mgmt/dm/inc/dmInt.h +++ b/source/dnode/mgmt/dm/inc/dmInt.h @@ -39,18 +39,25 @@ typedef struct SDnodeMgmt { SMgmtWrapper *pWrapper; } SDnodeMgmt; +// dmInt.c +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); +void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg); + // dmFile.c int32_t dmReadFile(SDnodeMgmt *pMgmt); int32_t dmWriteFile(SDnodeMgmt *pMgmt); void dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps); -// dmMsg.c +// dmHandle.c void dmInitMsgHandle(SMgmtWrapper *pWrapper); void dmSendStatusReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg); +int32_t dmProcessCDnodeMsg(SDnode *pDnode, SNodeMsg *pMsg); // dmWorker.c int32_t dmStartThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/dm/src/dmHandle.c b/source/dnode/mgmt/dm/src/dmHandle.c index 07eca3f230..6492995c5c 100644 --- a/source/dnode/mgmt/dm/src/dmHandle.c +++ b/source/dnode/mgmt/dm/src/dmHandle.c @@ -118,7 +118,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { } -static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { +static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { dndReleaseWrapper(pWrapper); @@ -146,7 +146,7 @@ static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg return code; } -static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { +static int32_t dmProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) { SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype); if (pWrapper == NULL) { terrno = TSDB_CODE_NODE_NOT_DEPLOYED; @@ -171,24 +171,24 @@ static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg * return code; } -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { +int32_t dmProcessCDnodeMsg(SDnode *pDnode, SNodeMsg *pMsg) { switch (pMsg->rpcMsg.msgType) { case TDMT_DND_CREATE_MNODE: - return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, MNODE, pMsg); case TDMT_DND_DROP_MNODE: - return dndProcessDropNodeMsg(pDnode, MNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, MNODE, pMsg); case TDMT_DND_CREATE_QNODE: - return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, QNODE, pMsg); case TDMT_DND_DROP_QNODE: - return dndProcessDropNodeMsg(pDnode, QNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, QNODE, pMsg); case TDMT_DND_CREATE_SNODE: - return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, SNODE, pMsg); case TDMT_DND_DROP_SNODE: - return dndProcessDropNodeMsg(pDnode, SNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, SNODE, pMsg); case TDMT_DND_CREATE_BNODE: - return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg); + return dmProcessCreateNodeMsg(pDnode, BNODE, pMsg); case TDMT_DND_DROP_BNODE: - return dndProcessDropNodeMsg(pDnode, BNODE, pMsg); + return dmProcessDropNodeMsg(pDnode, BNODE, pMsg); default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; return -1; diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index 1e38b8304d..c710af9006 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -78,7 +78,7 @@ static int32_t dmStart(SMgmtWrapper *pWrapper) { return dmStartThread(pWrapper->pMgmt); } -int32_t dmInit(SMgmtWrapper *pWrapper) { +static int32_t dmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt)); dInfo("dnode-mgmt start to init"); @@ -124,7 +124,7 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return 0; } -void dmCleanup(SMgmtWrapper *pWrapper) { +static void dmCleanup(SMgmtWrapper *pWrapper) { SDnodeMgmt *pMgmt = pWrapper->pMgmt; if (pMgmt == NULL) return; @@ -153,12 +153,12 @@ void dmCleanup(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt is cleaned up"); } -int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { +static int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) { *required = true; return 0; } -void dmGetMgmtFp(SMgmtWrapper *pWrapper) { +void dmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = dmInit; mgmtFp.closeFp = dmCleanup; diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/src/dmWorker.c index 6061189ca1..40ca858183 100644 --- a/source/dnode/mgmt/dm/src/dmWorker.c +++ b/source/dnode/mgmt/dm/src/dmWorker.c @@ -66,16 +66,6 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { dTrace("msg:%p, will be processed in dnode queue", pMsg); switch (pRpc->msgType) { - case TDMT_DND_CREATE_MNODE: - case TDMT_DND_CREATE_QNODE: - case TDMT_DND_CREATE_SNODE: - case TDMT_DND_CREATE_BNODE: - case TDMT_DND_DROP_MNODE: - case TDMT_DND_DROP_QNODE: - case TDMT_DND_DROP_SNODE: - case TDMT_DND_DROP_BNODE: - code = dndProcessNodeMsg(pMgmt->pDnode, pMsg); - break; case TDMT_DND_CONFIG_DNODE: code = dmProcessConfigReq(pMgmt, pMsg); break; @@ -89,8 +79,8 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { code = dmProcessGrantRsp(pMgmt, pMsg); break; default: - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType)); + code = dmProcessCDnodeMsg(pMgmt->pDnode, pMsg); + break; } if (pRpc->msgType & 1u) { diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index b324a279bc..449c4399db 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -27,13 +27,13 @@ #include "tlockfree.h" #include "tlog.h" #include "tmsg.h" +#include "tmsgcb.h" #include "tprocess.h" #include "tqueue.h" #include "trpc.h" #include "tthread.h" #include "ttime.h" #include "tworker.h" -#include "tmsgcb.h" #include "dnode.h" #include "monitor.h" @@ -42,12 +42,42 @@ extern "C" { #endif -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} +#define dFatal(...) \ + { \ + if (dDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ + } \ + } +#define dError(...) \ + { \ + if (dDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + } \ + } +#define dWarn(...) \ + { \ + if (dDebugFlag & DEBUG_WARN) { \ + taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ + } \ + } +#define dInfo(...) \ + { \ + if (dDebugFlag & DEBUG_INFO) { \ + taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); \ + } \ + } +#define dDebug(...) \ + { \ + if (dDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); \ + } \ + } +#define dTrace(...) \ + { \ + if (dDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); \ + } \ + } typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus; @@ -73,7 +103,7 @@ typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required); typedef struct SMsgHandle { - SMgmtWrapper *pQndWrapper; + SMgmtWrapper *pQndWrapper; SMgmtWrapper *pMndWrapper; SMgmtWrapper *pWrapper; } SMsgHandle; @@ -146,14 +176,11 @@ void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); int32_t dndMarkWrapper(SMgmtWrapper *pWrapper); void dndReleaseWrapper(SMgmtWrapper *pWrapper); +void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); // dndMonitor.h void dndSendMonitorReport(SDnode *pDnode); -// dndMsg.h -void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); - // dndStr.h const char *dndStatStr(EDndStatus stat); const char *dndNodeLogStr(ENodeType ntype); @@ -168,12 +195,12 @@ SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper); int32_t dndInitMsgHandle(SDnode *pDnode); // mgmt -void dmGetMgmtFp(SMgmtWrapper *pWrapper); -void bmGetMgmtFp(SMgmtWrapper *pWrapper); -void qmGetMgmtFp(SMgmtWrapper *pMgmt); -void smGetMgmtFp(SMgmtWrapper *pWrapper); -void vmGetMgmtFp(SMgmtWrapper *pWrapper); -void mmGetMgmtFp(SMgmtWrapper *pMgmt); +void dmSetMgmtFp(SMgmtWrapper *pWrapper); +void bmSetMgmtFp(SMgmtWrapper *pWrapper); +void qmSetMgmtFp(SMgmtWrapper *pMgmt); +void smSetMgmtFp(SMgmtWrapper *pWrapper); +void vmSetMgmtFp(SMgmtWrapper *pWrapper); +void mmSetMgmtFp(SMgmtWrapper *pMgmt); void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet); @@ -194,8 +221,8 @@ void vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads); int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo); void vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo); int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, - SMonGrantInfo *pGrantInfo); - + SMonGrantInfo *pGrantInfo); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 72feed7c62..80fefdb3ef 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -82,12 +82,12 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { } dndSetStatus(pDnode, DND_STAT_INIT); - dmGetMgmtFp(&pDnode->wrappers[DNODE]); - mmGetMgmtFp(&pDnode->wrappers[MNODE]); - vmGetMgmtFp(&pDnode->wrappers[VNODES]); - qmGetMgmtFp(&pDnode->wrappers[QNODE]); - smGetMgmtFp(&pDnode->wrappers[SNODE]); - bmGetMgmtFp(&pDnode->wrappers[BNODE]); + dmSetMgmtFp(&pDnode->wrappers[DNODE]); + mmSetMgmtFp(&pDnode->wrappers[MNODE]); + vmSetMgmtFp(&pDnode->wrappers[VNODES]); + qmSetMgmtFp(&pDnode->wrappers[QNODE]); + smSetMgmtFp(&pDnode->wrappers[SNODE]); + bmSetMgmtFp(&pDnode->wrappers[BNODE]); for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/src/mmInt.c index bd7446780d..97fdafe139 100644 --- a/source/dnode/mgmt/mm/src/mmInt.c +++ b/source/dnode/mgmt/mm/src/mmInt.c @@ -227,7 +227,7 @@ static int32_t mmStart(SMgmtWrapper *pWrapper) { return mndStart(pMgmt->pMnode); } -void mmGetMgmtFp(SMgmtWrapper *pWrapper) { +void mmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = mmOpen; mgmtFp.closeFp = mmClose; diff --git a/source/dnode/mgmt/qm/src/qmInt.c b/source/dnode/mgmt/qm/src/qmInt.c index 8079859b96..53c43bf841 100644 --- a/source/dnode/mgmt/qm/src/qmInt.c +++ b/source/dnode/mgmt/qm/src/qmInt.c @@ -112,7 +112,7 @@ int32_t qmOpen(SMgmtWrapper *pWrapper) { return code; } -void qmGetMgmtFp(SMgmtWrapper *pWrapper) { +void qmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = qmOpen; mgmtFp.closeFp = qmClose; diff --git a/source/dnode/mgmt/sm/src/smInt.c b/source/dnode/mgmt/sm/src/smInt.c index 6d2e2aaefd..a279655609 100644 --- a/source/dnode/mgmt/sm/src/smInt.c +++ b/source/dnode/mgmt/sm/src/smInt.c @@ -109,7 +109,7 @@ int32_t smOpen(SMgmtWrapper *pWrapper) { return code; } -void smGetMgmtFp(SMgmtWrapper *pWrapper) { +void smSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = smOpen; mgmtFp.closeFp = smClose; diff --git a/source/dnode/mgmt/vm/src/vmInt.c b/source/dnode/mgmt/vm/src/vmInt.c index d9ef7b5ae6..d78a567b69 100644 --- a/source/dnode/mgmt/vm/src/vmInt.c +++ b/source/dnode/mgmt/vm/src/vmInt.c @@ -333,7 +333,7 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { return 0; } -void vmGetMgmtFp(SMgmtWrapper *pWrapper) { +void vmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = vmInit; mgmtFp.closeFp = vmCleanup; From e41043074bff1643c73ed98d70caf05782152749 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Apr 2022 08:20:26 -0400 Subject: [PATCH 32/42] integrate constant calculate --- include/libs/function/functionMgt.h | 1 + source/libs/function/src/functionMgt.c | 4 ++++ source/libs/nodes/src/nodesCodeFuncs.c | 5 +++-- source/libs/parser/src/parCalcConst.c | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 7d46b543cb..207e936801 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -127,6 +127,7 @@ bool fmIsStringFunc(int32_t funcId); bool fmIsDatetimeFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); +bool fmIsPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 4034a0eb0f..5bd7f22987 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -104,6 +104,10 @@ bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsPseudoColumnFunc(int32_t funcId) { + return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); +} + bool fmIsWindowPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC); } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e39a7ecfa6..1b16ba9206 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1491,6 +1491,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkValueGenByCalc = "GenByCalc"; static const char* jkValueLiteral = "Literal"; static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; @@ -1544,7 +1545,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkValueTranslate, pNode->genByCalc); + code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc); } if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); @@ -1617,7 +1618,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->genByCalc); + code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc); } if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index a572ccf5bd..46cdcc4243 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "functionMgt.h" #include "parInt.h" #include "scalar.h" @@ -44,6 +45,9 @@ static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { SFunctionNode* pFunc = *pNode; + if (fmIsPseudoColumnFunc(pFunc->funcId)) { + return DEAL_RES_CONTINUE; + } SNode* pParam = NULL; FOREACH(pParam, pFunc->pParameterList) { if (QUERY_NODE_VALUE != nodeType(pParam)) { From 1839793b143b0c81844d0e15a595b46a2d41af2a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Apr 2022 20:31:11 +0800 Subject: [PATCH 33/42] rename files --- source/dnode/mgmt/CMakeLists.txt | 26 +++++++------------ source/dnode/mgmt/bm/{src => }/bmHandle.c | 0 source/dnode/mgmt/bm/{src => }/bmInt.c | 0 source/dnode/mgmt/bm/{src => }/bmWorker.c | 0 source/dnode/mgmt/dm/{src => }/dmFile.c | 0 source/dnode/mgmt/dm/{src => }/dmHandle.c | 0 source/dnode/mgmt/dm/{src => }/dmInt.c | 0 source/dnode/mgmt/dm/{src => }/dmWorker.c | 0 source/dnode/mgmt/{main => }/exe/dndMain.c | 0 source/dnode/mgmt/{bm => }/inc/bmInt.h | 0 source/dnode/mgmt/{dm => }/inc/dmInt.h | 0 source/dnode/mgmt/{main => }/inc/dnd.h | 0 source/dnode/mgmt/{main => }/inc/dndInt.h | 0 source/dnode/mgmt/{mm => }/inc/mmInt.h | 0 source/dnode/mgmt/{qm => }/inc/qmInt.h | 0 source/dnode/mgmt/{sm => }/inc/smInt.h | 0 source/dnode/mgmt/{vm => }/inc/vmInt.h | 0 source/dnode/mgmt/main/{src => }/dndEnv.c | 0 source/dnode/mgmt/main/{src => }/dndExec.c | 0 source/dnode/mgmt/main/{src => }/dndFile.c | 0 source/dnode/mgmt/main/{src => }/dndInt.c | 0 source/dnode/mgmt/main/{src => }/dndMonitor.c | 0 .../dnode/mgmt/main/{src => }/dndTransport.c | 0 source/dnode/mgmt/mm/{src => }/mmFile.c | 0 source/dnode/mgmt/mm/{src => }/mmHandle.c | 0 source/dnode/mgmt/mm/{src => }/mmInt.c | 0 source/dnode/mgmt/mm/{src => }/mmWorker.c | 0 source/dnode/mgmt/qm/{src => }/qmHandle.c | 0 source/dnode/mgmt/qm/{src => }/qmInt.c | 0 source/dnode/mgmt/qm/{src => }/qmWorker.c | 0 source/dnode/mgmt/sm/{src => }/smHandle.c | 0 source/dnode/mgmt/sm/{src => }/smInt.c | 0 source/dnode/mgmt/sm/{src => }/smWorker.c | 0 source/dnode/mgmt/vm/{src => }/vmFile.c | 0 source/dnode/mgmt/vm/{src => }/vmHandle.c | 0 source/dnode/mgmt/vm/{src => }/vmInt.c | 0 source/dnode/mgmt/vm/{src => }/vmWorker.c | 0 37 files changed, 10 insertions(+), 16 deletions(-) rename source/dnode/mgmt/bm/{src => }/bmHandle.c (100%) rename source/dnode/mgmt/bm/{src => }/bmInt.c (100%) rename source/dnode/mgmt/bm/{src => }/bmWorker.c (100%) rename source/dnode/mgmt/dm/{src => }/dmFile.c (100%) rename source/dnode/mgmt/dm/{src => }/dmHandle.c (100%) rename source/dnode/mgmt/dm/{src => }/dmInt.c (100%) rename source/dnode/mgmt/dm/{src => }/dmWorker.c (100%) rename source/dnode/mgmt/{main => }/exe/dndMain.c (100%) rename source/dnode/mgmt/{bm => }/inc/bmInt.h (100%) rename source/dnode/mgmt/{dm => }/inc/dmInt.h (100%) rename source/dnode/mgmt/{main => }/inc/dnd.h (100%) rename source/dnode/mgmt/{main => }/inc/dndInt.h (100%) rename source/dnode/mgmt/{mm => }/inc/mmInt.h (100%) rename source/dnode/mgmt/{qm => }/inc/qmInt.h (100%) rename source/dnode/mgmt/{sm => }/inc/smInt.h (100%) rename source/dnode/mgmt/{vm => }/inc/vmInt.h (100%) rename source/dnode/mgmt/main/{src => }/dndEnv.c (100%) rename source/dnode/mgmt/main/{src => }/dndExec.c (100%) rename source/dnode/mgmt/main/{src => }/dndFile.c (100%) rename source/dnode/mgmt/main/{src => }/dndInt.c (100%) rename source/dnode/mgmt/main/{src => }/dndMonitor.c (100%) rename source/dnode/mgmt/main/{src => }/dndTransport.c (100%) rename source/dnode/mgmt/mm/{src => }/mmFile.c (100%) rename source/dnode/mgmt/mm/{src => }/mmHandle.c (100%) rename source/dnode/mgmt/mm/{src => }/mmInt.c (100%) rename source/dnode/mgmt/mm/{src => }/mmWorker.c (100%) rename source/dnode/mgmt/qm/{src => }/qmHandle.c (100%) rename source/dnode/mgmt/qm/{src => }/qmInt.c (100%) rename source/dnode/mgmt/qm/{src => }/qmWorker.c (100%) rename source/dnode/mgmt/sm/{src => }/smHandle.c (100%) rename source/dnode/mgmt/sm/{src => }/smInt.c (100%) rename source/dnode/mgmt/sm/{src => }/smWorker.c (100%) rename source/dnode/mgmt/vm/{src => }/vmFile.c (100%) rename source/dnode/mgmt/vm/{src => }/vmHandle.c (100%) rename source/dnode/mgmt/vm/{src => }/vmInt.c (100%) rename source/dnode/mgmt/vm/{src => }/vmWorker.c (100%) diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 81c434eb27..0c9a3733a2 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,10 +1,10 @@ -aux_source_directory(dm/src DNODE_SRC) -aux_source_directory(qm/src DNODE_SRC) -aux_source_directory(bm/src DNODE_SRC) -aux_source_directory(sm/src DNODE_SRC) -aux_source_directory(vm/src DNODE_SRC) -aux_source_directory(mm/src DNODE_SRC) -aux_source_directory(main/src DNODE_SRC) +aux_source_directory(dm DNODE_SRC) +aux_source_directory(qm DNODE_SRC) +aux_source_directory(bm DNODE_SRC) +aux_source_directory(sm DNODE_SRC) +aux_source_directory(vm DNODE_SRC) +aux_source_directory(mm DNODE_SRC) +aux_source_directory(main DNODE_SRC) add_library(dnode STATIC ${DNODE_SRC}) target_link_libraries( dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor @@ -12,20 +12,14 @@ target_link_libraries( target_include_directories( dnode PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -aux_source_directory(main/exe EXEC_SRC) +aux_source_directory(exe EXEC_SRC) add_executable(taosd ${EXEC_SRC}) target_include_directories( taosd - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries(taosd dnode) diff --git a/source/dnode/mgmt/bm/src/bmHandle.c b/source/dnode/mgmt/bm/bmHandle.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmHandle.c rename to source/dnode/mgmt/bm/bmHandle.c diff --git a/source/dnode/mgmt/bm/src/bmInt.c b/source/dnode/mgmt/bm/bmInt.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmInt.c rename to source/dnode/mgmt/bm/bmInt.c diff --git a/source/dnode/mgmt/bm/src/bmWorker.c b/source/dnode/mgmt/bm/bmWorker.c similarity index 100% rename from source/dnode/mgmt/bm/src/bmWorker.c rename to source/dnode/mgmt/bm/bmWorker.c diff --git a/source/dnode/mgmt/dm/src/dmFile.c b/source/dnode/mgmt/dm/dmFile.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmFile.c rename to source/dnode/mgmt/dm/dmFile.c diff --git a/source/dnode/mgmt/dm/src/dmHandle.c b/source/dnode/mgmt/dm/dmHandle.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmHandle.c rename to source/dnode/mgmt/dm/dmHandle.c diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/dmInt.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmInt.c rename to source/dnode/mgmt/dm/dmInt.c diff --git a/source/dnode/mgmt/dm/src/dmWorker.c b/source/dnode/mgmt/dm/dmWorker.c similarity index 100% rename from source/dnode/mgmt/dm/src/dmWorker.c rename to source/dnode/mgmt/dm/dmWorker.c diff --git a/source/dnode/mgmt/main/exe/dndMain.c b/source/dnode/mgmt/exe/dndMain.c similarity index 100% rename from source/dnode/mgmt/main/exe/dndMain.c rename to source/dnode/mgmt/exe/dndMain.c diff --git a/source/dnode/mgmt/bm/inc/bmInt.h b/source/dnode/mgmt/inc/bmInt.h similarity index 100% rename from source/dnode/mgmt/bm/inc/bmInt.h rename to source/dnode/mgmt/inc/bmInt.h diff --git a/source/dnode/mgmt/dm/inc/dmInt.h b/source/dnode/mgmt/inc/dmInt.h similarity index 100% rename from source/dnode/mgmt/dm/inc/dmInt.h rename to source/dnode/mgmt/inc/dmInt.h diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/inc/dnd.h similarity index 100% rename from source/dnode/mgmt/main/inc/dnd.h rename to source/dnode/mgmt/inc/dnd.h diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/inc/dndInt.h similarity index 100% rename from source/dnode/mgmt/main/inc/dndInt.h rename to source/dnode/mgmt/inc/dndInt.h diff --git a/source/dnode/mgmt/mm/inc/mmInt.h b/source/dnode/mgmt/inc/mmInt.h similarity index 100% rename from source/dnode/mgmt/mm/inc/mmInt.h rename to source/dnode/mgmt/inc/mmInt.h diff --git a/source/dnode/mgmt/qm/inc/qmInt.h b/source/dnode/mgmt/inc/qmInt.h similarity index 100% rename from source/dnode/mgmt/qm/inc/qmInt.h rename to source/dnode/mgmt/inc/qmInt.h diff --git a/source/dnode/mgmt/sm/inc/smInt.h b/source/dnode/mgmt/inc/smInt.h similarity index 100% rename from source/dnode/mgmt/sm/inc/smInt.h rename to source/dnode/mgmt/inc/smInt.h diff --git a/source/dnode/mgmt/vm/inc/vmInt.h b/source/dnode/mgmt/inc/vmInt.h similarity index 100% rename from source/dnode/mgmt/vm/inc/vmInt.h rename to source/dnode/mgmt/inc/vmInt.h diff --git a/source/dnode/mgmt/main/src/dndEnv.c b/source/dnode/mgmt/main/dndEnv.c similarity index 100% rename from source/dnode/mgmt/main/src/dndEnv.c rename to source/dnode/mgmt/main/dndEnv.c diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/dndExec.c similarity index 100% rename from source/dnode/mgmt/main/src/dndExec.c rename to source/dnode/mgmt/main/dndExec.c diff --git a/source/dnode/mgmt/main/src/dndFile.c b/source/dnode/mgmt/main/dndFile.c similarity index 100% rename from source/dnode/mgmt/main/src/dndFile.c rename to source/dnode/mgmt/main/dndFile.c diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/dndInt.c similarity index 100% rename from source/dnode/mgmt/main/src/dndInt.c rename to source/dnode/mgmt/main/dndInt.c diff --git a/source/dnode/mgmt/main/src/dndMonitor.c b/source/dnode/mgmt/main/dndMonitor.c similarity index 100% rename from source/dnode/mgmt/main/src/dndMonitor.c rename to source/dnode/mgmt/main/dndMonitor.c diff --git a/source/dnode/mgmt/main/src/dndTransport.c b/source/dnode/mgmt/main/dndTransport.c similarity index 100% rename from source/dnode/mgmt/main/src/dndTransport.c rename to source/dnode/mgmt/main/dndTransport.c diff --git a/source/dnode/mgmt/mm/src/mmFile.c b/source/dnode/mgmt/mm/mmFile.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmFile.c rename to source/dnode/mgmt/mm/mmFile.c diff --git a/source/dnode/mgmt/mm/src/mmHandle.c b/source/dnode/mgmt/mm/mmHandle.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmHandle.c rename to source/dnode/mgmt/mm/mmHandle.c diff --git a/source/dnode/mgmt/mm/src/mmInt.c b/source/dnode/mgmt/mm/mmInt.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmInt.c rename to source/dnode/mgmt/mm/mmInt.c diff --git a/source/dnode/mgmt/mm/src/mmWorker.c b/source/dnode/mgmt/mm/mmWorker.c similarity index 100% rename from source/dnode/mgmt/mm/src/mmWorker.c rename to source/dnode/mgmt/mm/mmWorker.c diff --git a/source/dnode/mgmt/qm/src/qmHandle.c b/source/dnode/mgmt/qm/qmHandle.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmHandle.c rename to source/dnode/mgmt/qm/qmHandle.c diff --git a/source/dnode/mgmt/qm/src/qmInt.c b/source/dnode/mgmt/qm/qmInt.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmInt.c rename to source/dnode/mgmt/qm/qmInt.c diff --git a/source/dnode/mgmt/qm/src/qmWorker.c b/source/dnode/mgmt/qm/qmWorker.c similarity index 100% rename from source/dnode/mgmt/qm/src/qmWorker.c rename to source/dnode/mgmt/qm/qmWorker.c diff --git a/source/dnode/mgmt/sm/src/smHandle.c b/source/dnode/mgmt/sm/smHandle.c similarity index 100% rename from source/dnode/mgmt/sm/src/smHandle.c rename to source/dnode/mgmt/sm/smHandle.c diff --git a/source/dnode/mgmt/sm/src/smInt.c b/source/dnode/mgmt/sm/smInt.c similarity index 100% rename from source/dnode/mgmt/sm/src/smInt.c rename to source/dnode/mgmt/sm/smInt.c diff --git a/source/dnode/mgmt/sm/src/smWorker.c b/source/dnode/mgmt/sm/smWorker.c similarity index 100% rename from source/dnode/mgmt/sm/src/smWorker.c rename to source/dnode/mgmt/sm/smWorker.c diff --git a/source/dnode/mgmt/vm/src/vmFile.c b/source/dnode/mgmt/vm/vmFile.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmFile.c rename to source/dnode/mgmt/vm/vmFile.c diff --git a/source/dnode/mgmt/vm/src/vmHandle.c b/source/dnode/mgmt/vm/vmHandle.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmHandle.c rename to source/dnode/mgmt/vm/vmHandle.c diff --git a/source/dnode/mgmt/vm/src/vmInt.c b/source/dnode/mgmt/vm/vmInt.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmInt.c rename to source/dnode/mgmt/vm/vmInt.c diff --git a/source/dnode/mgmt/vm/src/vmWorker.c b/source/dnode/mgmt/vm/vmWorker.c similarity index 100% rename from source/dnode/mgmt/vm/src/vmWorker.c rename to source/dnode/mgmt/vm/vmWorker.c From e4404dc8e72bb8574ac83d61714e42ba029463d2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 21:02:58 +0800 Subject: [PATCH 34/42] add more log --- source/client/src/tmq.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 81219e1c36..6c0bdd4bab 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -140,6 +140,7 @@ typedef struct { tmq_t* tmq; SMqClientVg* pVg; int32_t epoch; + int32_t vgId; tsem_t rspSem; tmq_message_t** msg; int32_t sync; @@ -839,7 +840,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { - tscWarn("msg discard, code:%x", code); + tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code); goto CREATE_MSG_FAIL; } @@ -848,12 +849,12 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (msgEpoch < tmqEpoch) { /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/ /*tsem_post(&tmq->rspSem);*/ - tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); + tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); return 0; } if (msgEpoch != tmqEpoch) { - tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch); + tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); } else { atomic_sub_fetch_32(&tmq->waitingRequest, 1); } @@ -1041,7 +1042,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) { if (epStatus == 1) { int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); - if (epSkipCnt < 40) return 0; + if (epSkipCnt < 5000) return 0; } atomic_store_32(&tmq->epSkipCnt, 0); int32_t tlen = sizeof(SMqCMGetSubEpReq); @@ -1256,6 +1257,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { } pParam->tmq = tmq; pParam->pVg = pVg; + pParam->vgId = pVg->vgId; pParam->epoch = tmq->epoch; pParam->sync = 0; @@ -1282,7 +1284,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ atomic_add_fetch_32(&tmq->waitingRequest, 1); - tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset); + tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; From ac82b91ed17759cdc5ef819e8877ffae0d32a3e2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 2 Apr 2022 23:00:12 +0800 Subject: [PATCH 35/42] handle except --- source/libs/transport/src/transSrv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index dfb5eb35d6..79c72b3a35 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -193,6 +193,7 @@ static void uvHandleReq(SSrvConn* pConn) { transMsg.ahandle = (void*)pHead->ahandle; transMsg.handle = NULL; + // transDestroyBuffer(&pConn->readBuf); transClearBuffer(&pConn->readBuf); pConn->inType = pHead->msgType; if (pConn->status == ConnNormal) { @@ -249,6 +250,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { conn->broken = true; if (conn->status == ConnAcquire) { if (conn->regArg.init) { + tTrace("server conn %p broken, notify server app", conn); STrans* pTransInst = conn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); memset(&conn->regArg, 0, sizeof(conn->regArg)); @@ -270,7 +272,7 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnSendCb(uv_write_t* req, int status) { SSrvConn* conn = req->data; - transClearBuffer(&conn->readBuf); + // transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("server conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { From 798c3a9f825239103f572b5ca0e2bbe07a59b75f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 23:03:12 +0800 Subject: [PATCH 36/42] add more log --- source/client/src/tmq.c | 12 ++++++----- source/dnode/mnode/impl/src/mndConsumer.c | 4 +--- source/dnode/mnode/impl/src/mndScheduler.c | 3 +++ source/dnode/mnode/impl/src/mndSubscribe.c | 24 ++++++++++++++++------ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 6c0bdd4bab..602ecdf6ab 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -917,10 +917,10 @@ CREATE_MSG_FAIL: bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ - tscDebug("consumer %ld update ep epoch %d to epoch %d", tmq->consumerId, tmq->epoch, epoch); bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { return false; @@ -938,17 +938,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { taosHashClear(pHash); topic.topicName = strdup(pTopicEp->topic); + tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int32_t j = 0; j < topicNumCur; j++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); + tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); if (vgNumCur == 0) break; for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer %ld epoch %d vg %d build %s\n", tmq->consumerId, epoch, pVgCur->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -962,10 +964,10 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("consumer %ld epoch %d vg %d offset og to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("consumer %ld epoch %d vg %d found %s\n", tmq->consumerId, epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); } tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { @@ -1231,7 +1233,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1); - tscDebug("consumer %ld skip vg %d skip cnt %ld", tmq->consumerId, pVg->vgId, skipCnt); + tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt); continue; #if 0 if (skipCnt < 30000) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index c4c93b7fc9..6080ec7710 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -160,10 +160,8 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId); - pOldConsumer->epoch++; - - // TODO handle update /*taosWLockLatch(&pOldConsumer->lock);*/ + atomic_add_fetch_32(&pOldConsumer->epoch, 1); /*taosWUnLockLatch(&pOldConsumer->lock);*/ return 0; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 4562d9e5d3..df7946a0c1 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -471,6 +471,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib consumerEp.consumerId = -1; consumerEp.epSet = plan->execNode.epSet; consumerEp.vgId = plan->execNode.nodeId; + + mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId); + int32_t msgLen; if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) { sdbRelease(pSdb, pVgroup); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index ae9a4198a4..57d98396ea 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -212,19 +212,24 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } + //TODO add lock ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + int32_t serverEpoch = pConsumer->epoch; // TODO int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mTrace("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, pConsumer->epoch, hbStatus); + mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, hbStatus); atomic_store_32(&pConsumer->hbStatus, 0); /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ strcpy(rsp.cgroup, pReq->cgroup); - if (epoch != pConsumer->epoch) { - mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch); + if (epoch != serverEpoch) { + mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, serverEpoch); + mDebug("consumer %ld try r lock", consumerId); + taosRLockLatch(&pConsumer->lock); + mDebug("consumer %ld r locked", consumerId); SArray *pTopics = pConsumer->currentTopics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -238,7 +243,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); if (consumerId == pSubConsumer->consumerId) { int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - mInfo("topic %s has %d vg", topicName, pConsumer->epoch); + mInfo("topic %s has %d vg", topicName, serverEpoch); SMqSubTopicEp topicEp; strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); @@ -264,6 +269,8 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { } mndReleaseSubscribe(pMnode, pSub); } + taosRUnLockLatch(&pConsumer->lock); + mDebug("consumer %ld r unlock", consumerId); } int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); @@ -272,7 +279,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { return -1; } ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = pConsumer->epoch; + ((SMqRspHead *)buf)->epoch = serverEpoch; ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -395,7 +402,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); taosMemoryFreeClear(pRebSub->key); - mInfo("mq rebalance subscription: %s", pSub->key); + mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, (int32_t)taosArrayGetSize(pSub->unassignedVg)); // remove lost consumer for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { @@ -442,6 +449,9 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { } SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); + mDebug("consumer %ld try w lock", pRebConsumer->consumerId); + taosWLockLatch(&pRebConsumer->lock); + mDebug("consumer %ld w locked", pRebConsumer->consumerId); int32_t status = atomic_load_32(&pRebConsumer->status); if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || @@ -462,6 +472,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); mndTransAppendCommitlog(pTrans, pConsumerRaw); } + taosWUnLockLatch(&pRebConsumer->lock); + mDebug("consumer %ld w unlock", pRebConsumer->consumerId); mndReleaseConsumer(pMnode, pRebConsumer); } From 5298436e1f67e193d3169d5745f9eb285cda903a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Apr 2022 23:20:11 +0800 Subject: [PATCH 37/42] add test for multi vgroup consume --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b756976394..5a3ee003f0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -41,7 +41,7 @@ # ---- tmq ./test.sh -f tsim/tmq/basic.sim ./test.sh -f tsim/tmq/basic1.sim -#./test.sh -f tsim/tmq/oneTopic.sim +./test.sh -f tsim/tmq/oneTopic.sim #./test.sh -f tsim/tmq/multiTopic.sim # --- stable From b3bb652708c0e50d88d0821db9673914e8c6db42 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 18:02:30 +0800 Subject: [PATCH 38/42] data and bitmap compress/store/decompress separatedly --- include/common/tdataformat.h | 5 +- include/common/trow.h | 28 ++++----- source/common/src/tdataformat.c | 26 +++++--- source/common/src/trow.c | 2 +- source/dnode/vnode/src/inc/tsdbReadImpl.h | 8 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 33 ++++++++-- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 73 +++++++++++++++------- source/util/src/tcompression.c | 4 +- 8 files changed, 120 insertions(+), 59 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index da724206d9..991cb55e50 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -135,8 +135,8 @@ typedef struct { #define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 #define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) -#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) +#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS)) +#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8)) int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); @@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche } #endif // ----------------- Data column structure +// SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { int8_t type; // column type uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM diff --git a/include/common/trow.h b/include/common/trow.h index 01f4076382..7cde8c50c5 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -147,20 +147,20 @@ typedef struct { typedef struct { // basic info - int8_t rowType; - int16_t sver; - STSRow *pBuf; + int8_t rowType; + schema_ver_t sver; + STSRow *pBuf; // extended info - int32_t flen; - int16_t nBoundCols; - int16_t nCols; - int16_t nBitmaps; - int16_t nBoundBitmaps; - int32_t offset; - void *pBitmap; - void *pOffset; - int32_t extendedRowSize; + int32_t flen; + col_id_t nBoundCols; + col_id_t nCols; + col_id_t nBitmaps; + col_id_t nBoundBitmaps; + int32_t offset; + void *pBitmap; + void *pOffset; + int32_t extendedRowSize; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t } #ifdef TD_SUPPORT_BITMAP // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); } else { pBuilder->nBoundBitmaps = 0; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1d7a4aeb8f..c1225012ac 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += (int)nBitmapBytes; // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered. - spaceNeeded += TYPE_BYTES[pCol->type]; + // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove + // the additional space #endif if (pCol->spaceSize < spaceNeeded) { @@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { } } #ifdef TD_SUPPORT_BITMAP + if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints); pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes); @@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) bool isNEleNull(SDataCol *pCol, int nEle) { if (isAllRowsNull(pCol)) return true; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } return true; @@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { static void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { dataColSetNullAt(pCol, i); } } else { @@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { // char *tptr = (char *)(pCol->pData); VarDataOffsetT offset = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { pCol->dataOff[i] = offset; offset += varDataTLen(tptr); tptr = POINTER_SHIFT(tptr, varDataTLen(tptr)); @@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { tdFreeDataCols(pCols); return NULL; } +#if 0 // no need as calloc used int i; for (i = 0; i < maxCols; i++) { pCols->cols[i].spaceSize = 0; @@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; } +#endif } return pCols; @@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; - for (i = oldMaxCols; i < pCols->maxCols; i++) { + for (i = oldMaxCols; i < pCols->maxCols; ++i) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; + pCols->cols[i].pBitmap = NULL; pCols->cols[i].spaceSize = 0; } } +#if 0 + tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...) +#endif - tdResetDataCols(pCols); + pCols->numOfRows = 0; pCols->numOfCols = schemaNCols(pSchema); - for (i = 0; i < schemaNCols(pSchema); i++) { + for (i = 0; i < schemaNCols(pSchema); ++i) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } @@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { if (pCols) { if (pCols->cols) { int maxCols = pCols->maxCols; - for (i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; ++i) { SDataCol *pCol = &pCols->cols[i]; taosMemoryFreeClear(pCol->pData); } @@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; - for (int i = 0; i < pCols->maxCols; i++) { + for (int i = 0; i < pCols->maxCols; ++i) { dataColReset(pCols->cols + i); } } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9ee2ec1300..0d5a874c1c 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); } } diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 17c220a35a..90a877bb83 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -112,10 +112,10 @@ typedef struct { #else typedef struct { int16_t colId; - uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint8_t reserve : 7; - uint8_t type; - int32_t len; + uint16_t type : 6; + uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) + uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint32_t len : 31; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d5e9b55a71..497bae6ee1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish @@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; + int32_t tBitmapsLen = 0; if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tBitmaps = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tlen += tBitmaps; @@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif // move bitmap parts ahead // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove - memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); + // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); } #endif void *tptr; // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { return -1; } pBlockData = (SBlockData *)(*ppBuf); @@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF // Compress or just copy if (pCfg->compression) { +#if 0 flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr, tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tlen + COMP_OVERFLOW_BYTES); +#endif + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + if (tBitmaps > 0) { + tptr = POINTER_SHIFT(pBlockData, lsize + flen); + tBitmapsLen = + tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, rowsToWrite, tptr, tBitmaps + COMP_OVERFLOW_BYTES, + pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); + TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES))); + flen += tBitmapsLen; + } } else { flen = tlen; memcpy(tptr, pDataCol->pData, flen); + if (tBitmaps > 0) { + tptr = POINTER_SHIFT(pBlockData, lsize + flen); + memcpy(tptr, pDataCol->pBitmap, tBitmaps); + tBitmapsLen = tBitmaps; + flen += tBitmapsLen; + } } // Add checksum ASSERT(flen > 0); + ASSERT(tBitmapsLen <= 1024); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); if (ncol != 0) { tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; + pBlockCol->len = flen; // data + bitmaps + pBlockCol->blen = tBitmapsLen; ++tcol; } else { keyLen = flen; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 304b3286fe..8cb90d76a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, int numOfColIds); static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); @@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; continue; } @@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat TD_SET_COL_ROWS_NORM(pDataCol); } - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } if (tcolId == pDataCol->colId) { if (pBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, - pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), - (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen, + pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints, + TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; @@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return 0; } -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize) { +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) { if (!taosCheckChecksumWhole((uint8_t *)content, len)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; @@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 if (comp) { // Need to decompress int tlen = - (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps, + (*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); if (tlen <= 0) { - tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", - len, comp, numOfRows, maxPoints, bufferSize); + tsdbError( + "Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + (int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } pDataCol->len = tlen; + + if (numOfBitmaps > 0) { + tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps, + pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError( + "Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d " + "bufferSize:%d", + bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + // pDataCol->blen = tlen; + } } else { // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); + pDataCol->len = len - bitmapLen - sizeof(TSCKSUM); memcpy(pDataCol->pData, content, pDataCol->len); + if (numOfBitmaps > 0) { + // pDataCol->blen = bitmapLen; + memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen); + } } +#if 0 if (lenOfBitmaps > 0) { pDataCol->len -= lenOfBitmaps; @@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 } else if (IS_VAR_DATA_TYPE(pDataCol->type)) { dataColSetOffset(pDataCol, numOfRows); } - +#endif + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + dataColSetOffset(pDataCol, numOfRows); + } return 0; } @@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { ASSERT(pDataCol->colId == pBlockCol->colId); - STsdb * pRepo = TSDB_READ_REPO(pReadh); + STsdb *pRepo = TSDB_READ_REPO(pReadh); STsdbCfg *pCfg = REPO_CFG(pRepo); - int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); - int32_t tBitmaps = 0; + int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if (!TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } - int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; + int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; @@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows, - tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, + if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm, + pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), pBlockCol->colId, offset); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b2332aded7..041aec4054 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -28,11 +28,11 @@ * * BOOLEAN Compression Algorithm: * We provide two methods for compress boolean types. Because boolean types in C - * code are char bytes with 0 and 1 values only, only one bit can used to discrimenate + * code are char bytes with 0 and 1 values only, only one bit can used to discriminate * the values. * 1. The first method is using only 1 bit to represent the boolean value with 1 for * true and 0 for false. Then the compression rate is 1/8. - * 2. The second method is using run length encoding (RLE) methods. This methos works + * 2. The second method is using run length encoding (RLE) methods. This method works * better when there are a lot of consecutive true values or false values. * * STRING Compression Algorithm: From 489f65f2ce95954d1cb8c189bef51a0e5d2afa37 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 18:10:21 +0800 Subject: [PATCH 39/42] trigger CI From 670554dcdd7ec8f63dcdb8c56e16c75079c5670b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 18:56:44 +0800 Subject: [PATCH 40/42] bug fix --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 497bae6ee1..2a8875fefe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1315,7 +1315,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } #endif - void *tptr; + void *tptr, *bptr; // Make room if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { @@ -1340,9 +1340,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tlen + COMP_OVERFLOW_BYTES); if (tBitmaps > 0) { - tptr = POINTER_SHIFT(pBlockData, lsize + flen); + bptr = POINTER_SHIFT(pBlockData, lsize + flen); tBitmapsLen = - tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, rowsToWrite, tptr, tBitmaps + COMP_OVERFLOW_BYTES, + tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES))); flen += tBitmapsLen; @@ -1351,8 +1351,8 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF flen = tlen; memcpy(tptr, pDataCol->pData, flen); if (tBitmaps > 0) { - tptr = POINTER_SHIFT(pBlockData, lsize + flen); - memcpy(tptr, pDataCol->pBitmap, tBitmaps); + bptr = POINTER_SHIFT(pBlockData, lsize + flen); + memcpy(bptr, pDataCol->pBitmap, tBitmaps); tBitmapsLen = tBitmaps; flen += tBitmapsLen; } From 3ce4784a74b5ee6448ca0e484707b86c243b374d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 19:48:15 +0800 Subject: [PATCH 41/42] trigger CI From cea37a8e8f4880eb63865c15957b133f0a4f2705 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 20:28:12 +0800 Subject: [PATCH 42/42] trigger CI