From d57da814cc4863db4ef2c439e72a1b59aeb0f534 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Apr 2022 14:35:03 +0800 Subject: [PATCH 01/31] fix(query): fix timediff args cannot be integer values TD-14798 --- source/libs/function/src/builtins.c | 15 +++++++------ source/libs/scalar/src/sclfunc.c | 33 ----------------------------- 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index ed8cb6c21f..78dec26be5 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -330,7 +330,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l } uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_VAR_DATA_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { + if (!IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -358,7 +358,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; - if ((!IS_VAR_DATA_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || !IS_INTEGER_TYPE(para2Type)) { + if ((!IS_VAR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || !IS_INTEGER_TYPE(para2Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -372,12 +372,13 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; - if ((!IS_VAR_DATA_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || - (!IS_VAR_DATA_TYPE(para2Type) && TSDB_DATA_TYPE_TIMESTAMP != para2Type)) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + for (int32_t i = 0; i < 2; ++i) { + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; + if (!IS_VAR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } } + if (3 == paraNum) { if (!IS_INTEGER_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4c00dbad34..ad7341b45a 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -795,13 +795,6 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); - if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP) { - return TSDB_CODE_FAILED; - } - - if (inputNum != 1) { - return TSDB_CODE_FAILED; - } char *input = pInput[0].columnData->pData; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -867,13 +860,6 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); int32_t timePrec = GET_PARAM_PRECISON(pInput); - if (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { - return TSDB_CODE_FAILED; - } - - if (inputNum != 1) { - return TSDB_CODE_FAILED; - } char *input = pInput[0].columnData->pData + pInput[0].columnData->varmeta.offset[0]; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -897,18 +883,6 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(&pInput[0]); int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); - if (inputNum != 2) { - return TSDB_CODE_FAILED; - } - - if (type != TSDB_DATA_TYPE_BIGINT && type != TSDB_DATA_TYPE_TIMESTAMP && - type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR) { - return TSDB_CODE_FAILED; - } - - if (GET_PARAM_TYPE(&pInput[1]) != TSDB_DATA_TYPE_BIGINT) { //time_unit - return TSDB_CODE_FAILED; - } int64_t timeUnit, timeVal = 0; GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); @@ -1096,16 +1070,9 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 2 && inputNum != 3) { - return TSDB_CODE_FAILED; - } - int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); int64_t timeUnit = -1, timeVal[2] = {0}; if (inputNum == 3) { - if (GET_PARAM_TYPE(&pInput[2]) != TSDB_DATA_TYPE_BIGINT) { - return TSDB_CODE_FAILED; - } GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); } From f3b8bc8152ba599b8646cfe521f3ff5b02dc3cc5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Apr 2022 14:35:03 +0800 Subject: [PATCH 02/31] fix(query): fix timediff/timetruncate/to_unixtimestamp handling NULL value and empty string TD-14802 TD-14803 --- source/common/src/ttime.c | 12 ++++++++++-- source/libs/scalar/src/sclfunc.c | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 4686d856cc..2f3677330b 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -415,7 +415,11 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec if (type == TSDB_DATA_TYPE_BINARY) { newColData = taosMemoryCalloc(1, charLen + 1); memcpy(newColData, varDataVal(inputData), charLen); - taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); + bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0); + if (ret != TSDB_CODE_SUCCESS) { + taosMemoryFree(newColData); + return ret; + } taosMemoryFree(newColData); } else if (type == TSDB_DATA_TYPE_NCHAR) { newColData = taosMemoryCalloc(1, charLen / TSDB_NCHAR_SIZE + 1); @@ -425,7 +429,11 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec return TSDB_CODE_FAILED; } newColData[len] = 0; - taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0); + bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0); + if (ret != TSDB_CODE_SUCCESS) { + taosMemoryFree(newColData); + return ret; + } taosMemoryFree(newColData); } else { return TSDB_CODE_FAILED; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index ad7341b45a..e7ff0bde91 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -869,7 +869,11 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP } int64_t timeVal = 0; - convertStringToTimestamp(type, input, timePrec, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, timePrec, &timeVal); + if (ret != TSDB_CODE_SUCCESS) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } colDataAppend(pOutput->columnData, i, (char *)&timeVal, false); input += varDataTLen(input); @@ -904,7 +908,11 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal); + int32_t ret = convertStringToTimestamp(type, input, TSDB_TIME_PRECISION_NANO, &timeVal); + if (ret != TSDB_CODE_SUCCESS) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } //If converted value is less than 10digits in second, use value in second instead int64_t timeValSec = timeVal / 1000000000; if (timeValSec < 1000000000) { @@ -1100,7 +1108,11 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int32_t type = GET_PARAM_TYPE(&pInput[k]); if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ - convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); + int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); + if (ret != TSDB_CODE_SUCCESS) { + colDataAppendNULL(pOutput->columnData, i); + continue; + } } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/ GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]); if (type == TSDB_DATA_TYPE_TIMESTAMP) { From dc76f2e91ab0ab6ef1fcf8aa3b42ae067a0654a1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 17:32:50 +0800 Subject: [PATCH 03/31] refactor: refactor hash module to invoke the freeFp when destroying the hash node. --- source/libs/executor/src/executorimpl.c | 5 +---- source/util/src/thash.c | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ed2454faae..5261c182d3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -20,23 +20,19 @@ #include "tname.h" #include "os.h" -#include "parser.h" #include "tdatablock.h" -#include "texception.h" #include "tglobal.h" #include "tmsg.h" #include "tsort.h" #include "ttime.h" #include "executorimpl.h" -#include "function.h" #include "query.h" #include "tcompare.h" #include "tcompression.h" #include "thash.h" #include "vnode.h" #include "ttypes.h" -#include "vnode.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -4879,6 +4875,7 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasi if (!resultRow) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_INVALID_INPUT); } + // add a new result set for a new group SResultRowPosition pos = {.pageId = resultRow->pageId, .offset = resultRow->offset}; taosHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &pos, sizeof(SResultRowPosition)); diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 809d008aa7..4874bc16d7 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -31,9 +31,12 @@ #define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) #define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) -#define FREE_HASH_NODE(_n) \ - do { \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_fp, _n) \ + do { \ + if (_fp != NULL) { \ + (_fp)(_n); \ + } \ + taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { @@ -195,7 +198,7 @@ static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SH if (pNode->refCount <= 0) { pNewNode->next = pNode->next; - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pHashObj->freeFp, pNode); } else { pNewNode->next = pNode; pe->num++; @@ -310,7 +313,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo return -1; } - uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { @@ -523,7 +526,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pe->num--; atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pHashObj->freeFp, pNode); } } else { prevNode = pNode; @@ -558,7 +561,7 @@ void taosHashClear(SHashObj *pHashObj) { while (pNode) { pNext = pNode->next; - FREE_HASH_NODE(pNode); + FREE_HASH_NODE(pHashObj->freeFp, pNode); pNode = pNext; } @@ -769,7 +772,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { pe->num--; atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pOld); + FREE_HASH_NODE(pHashObj->freeFp, pOld); } } else { // uError("pNode:%p data:%p is not there!!!", pNode, p); From 315a6c0364d0cfc58603daf42d55e95be62b161b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Apr 2022 14:35:03 +0800 Subject: [PATCH 04/31] refactor(query): constant NULL value processing logic TD-14802 TD-14803 --- source/common/src/tdatablock.c | 5 +++++ source/libs/scalar/src/scalar.c | 27 +++++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index beabc1b6eb..14b7fe5d0e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1100,6 +1100,11 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) pColumn->nullbitmap = tmp; memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); + + if (pColumn->info.type == TSDB_DATA_TYPE_NULL) { + return TSDB_CODE_SUCCESS; + } + assert(pColumn->info.bytes); tmp = taosMemoryRealloc(pColumn->pData, numOfRows * pColumn->info.bytes); if (tmp == NULL) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 116dfdd5d5..0432ae1df8 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -398,7 +398,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SScalarParam *params = NULL; int32_t rowNum = 0; int32_t code = 0; - + SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); output->columnData = createColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { @@ -411,7 +411,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t paramNum = scalarGetOperatorParamNum(node->opType); SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; - + OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); _return: @@ -426,7 +426,7 @@ _return: EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SScalarParam output = {0}; - + ctx->code = sclExecFunction(node, ctx, &output); if (ctx->code) { return DEAL_RES_ERROR; @@ -440,16 +440,19 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - res->node.resType = node->node.resType; - - int32_t type = output.columnData->info.type; - if (IS_VAR_DATA_TYPE(type)) { - res->datum.p = output.columnData->pData; - output.columnData->pData = NULL; + if (colDataIsNull_s(output.columnData, 0)) { + res->node.resType.type = TSDB_DATA_TYPE_NULL; } else { - memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + res->node.resType = node->node.resType; + int32_t type = output.columnData->info.type; + if (IS_VAR_DATA_TYPE(type)) { + res->datum.p = output.columnData->pData; + output.columnData->pData = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); + } } - + nodesDestroyNode(*pNode); *pNode = (SNode*)res; @@ -469,7 +472,7 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - sclFreeParam(&output); + sclFreeParam(&output); ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } From 1c0211880ff15f64702154cfe64ee586c9d71245 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 18 Apr 2022 14:35:03 +0800 Subject: [PATCH 05/31] refactor(query): support timefunction arithmetic operation with boolean constant --- source/libs/parser/src/parTranslater.c | 6 ++++-- source/libs/scalar/inc/sclvector.h | 5 +++++ source/libs/scalar/src/sclvector.c | 12 ++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 361d1a3b29..45e8cd3acc 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -492,8 +492,10 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BIGINT == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BIGINT == ldt.type)) { + if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type) ) { pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } else { diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index e51115e9c0..f15116bdb7 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -52,6 +52,9 @@ static FORCE_INLINE double getVectorDoubleValue_FLOAT(void *src, int32_t index) static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { return (double)*((double *)src + index); } +static FORCE_INLINE double getVectorDoubleValue_BOOL(void *src, int32_t index) { + return (double)*((bool *)src + index); +} static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { _getDoubleValue_fn_t p = NULL; @@ -77,6 +80,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) p = getVectorDoubleValue_DOUBLE; } else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) { p = getVectorDoubleValue_BIGINT; + } else if (srcType == TSDB_DATA_TYPE_BOOL) { + p = getVectorDoubleValue_BOOL; } else { assert(0); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 253944f757..0131a94fec 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -57,6 +57,10 @@ int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) { int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) { return (int64_t)*((double *)src + index); } +int64_t getVectorBigintValue_BOOL(void *src, int32_t index) { + return (int64_t)*((bool *)src + index); +} + _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { _getBigintValue_fn_t p = NULL; if(srcType==TSDB_DATA_TYPE_TINYINT) { @@ -81,6 +85,8 @@ _getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { p = getVectorBigintValue_DOUBLE; }else if(srcType==TSDB_DATA_TYPE_TIMESTAMP) { p = getVectorBigintValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_BOOL) { + p = getVectorBigintValue_BOOL; }else { assert(0); } @@ -620,8 +626,10 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut SColumnInfoData *pLeftCol = doVectorConvert(pLeft, &leftConvert); SColumnInfoData *pRightCol = doVectorConvert(pRight, &rightConvert); - if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BIGINT) || - (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BIGINT)) { //timestamp plus duration + if ((GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pRight))) || + (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && IS_INTEGER_TYPE(GET_PARAM_TYPE(pLeft))) || + (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_BOOL) || + (GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_TIMESTAMP && GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BOOL)) { //timestamp plus duration int64_t *output = (int64_t *)pOutputCol->pData; _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); From 9864e367bba078a811becec35fcf504c42eaee23 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 18:47:59 +0800 Subject: [PATCH 06/31] refactor(query): do some internal refactor in executor. --- source/libs/executor/inc/executorimpl.h | 8 +-- source/libs/executor/src/executorimpl.c | 66 +++++++++++++++--------- source/libs/executor/src/groupoperator.c | 9 ++-- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5ef1754913..fa86cd2a18 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -342,7 +342,7 @@ typedef struct STableScanInfo { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; - double sampleRatio; // data block sample ratio + double sampleRatio; // data block sample ratio, 1 by default SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. } STableScanInfo; @@ -395,7 +395,6 @@ typedef struct SOptrBasicInfo { int32_t* rowCellInfoOffset; // offset value for each row result cell info SqlFunctionCtx* pCtx; SSDataBlock* pRes; - int32_t capacity; // TODO remove it } SOptrBasicInfo; // TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset @@ -422,6 +421,8 @@ typedef struct STableIntervalOperatorInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. SColumnInfoData timeWindowData; // query time window info for scalar function execution. + double watermark; // water mark + int32_t trigger; // trigger model } STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -602,7 +603,8 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); void operatorDummyCloseFn(void* param, int32_t numOfCols); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, - int32_t numOfRows, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); + SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey); +void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void toSDatablock(SSDataBlock* pBlock, int32_t rowCapacity, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, int32_t* rowCellOffset); void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5261c182d3..beeea4f223 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4449,7 +4449,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { SSortHandle* pHandle = pInfo->pSortHandle; SSDataBlock* pDataBlock = createOneDataBlock(pInfo->binfo.pRes, false); - blockDataEnsureCapacity(pDataBlock, pInfo->binfo.capacity); + blockDataEnsureCapacity(pDataBlock, pOperator->resultInfo.capacity); while (1) { blockDataCleanup(pDataBlock); @@ -4461,7 +4461,7 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) { // build datablock for merge for one group appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= pInfo->binfo.capacity) { + if (pDataBlock->info.rows >= pOperator->resultInfo.capacity) { break; } } @@ -4496,7 +4496,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSortedMergeOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->binfo.capacity); + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity); } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; @@ -4603,7 +4603,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pInfo->bufPageSize = 1024; pInfo->pSortInfo = pSortInfo; - pInfo->binfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize); + pOperator->resultInfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize); pOperator->name = "SortedMerge"; // pOperator->operatorType = OP_SortedMerge; @@ -5103,8 +5103,8 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro return NULL; } - blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); - toSDatablock(pInfo->binfo.pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { @@ -5124,7 +5124,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup } if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pInfo->binfo.pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -5158,8 +5158,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); - toSDatablock(pInfo->binfo.pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); ASSERT(pInfo->binfo.pRes->info.rows > 0); pOperator->status = OP_RES_TO_RETURN; @@ -5260,8 +5260,8 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); OPTR_SET_OPENED(pOperator); - blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); - toSDatablock(pInfo->binfo.pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + toSDatablock(pInfo->binfo.pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { @@ -5352,7 +5352,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pBInfo->pRes, pBInfo->capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5383,8 +5383,8 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); - blockDataEnsureCapacity(pBInfo->pRes, pBInfo->capacity); - toSDatablock(pBInfo->pRes, pBInfo->capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5401,7 +5401,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pBInfo->pRes, pBInfo->capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5432,8 +5432,8 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); - blockDataEnsureCapacity(pBInfo->pRes, pBInfo->capacity); - toSDatablock(pBInfo->pRes, pBInfo->capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); + blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); + toSDatablock(pBInfo->pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5611,15 +5611,23 @@ static void cleanupAggSup(SAggSupporter* pAggSup) { } int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, - int32_t numOfRows, SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey) { + SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey) { pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pRes = pResultBlock; - pBasicInfo->capacity = numOfRows; doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey); return TSDB_CODE_SUCCESS; } +void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows) { + pOperator->resultInfo.capacity = numOfRows; + pOperator->resultInfo.threshold = numOfRows * 0.75; + + if (pOperator->resultInfo.threshold == 0) { + pOperator->resultInfo.capacity = numOfRows; + } +} + static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInfo) { STableQueryInfo* pTableQueryInfo = taosMemoryCalloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pTableQueryInfo == NULL) { @@ -5655,7 +5663,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* int32_t numOfRows = 1; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResultBlock, keyBufSize, pTaskInfo->id.str); + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResultBlock, keyBufSize, pTaskInfo->id.str); pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); if (code != TSDB_CODE_SUCCESS || pInfo->pTableQueryInfo == NULL) { goto _error; @@ -5805,7 +5815,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p int32_t numOfCols = num; int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, keyBufSize, pTaskInfo->id.str); + + initResultSizeInfo(pOperator, numOfRows); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols); @@ -5854,7 +5866,9 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, keyBufSize, pTaskInfo->id.str); + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->timeWindowData, &pInfo->win); // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); @@ -5933,7 +5947,9 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pInfo->colIndex = -1; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, 4096, pResBlock, keyBufSize, pTaskInfo->id.str); + + initResultSizeInfo(pOperator, 4096); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pOperator->name = "StateWindowOperator"; @@ -5968,7 +5984,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, numOfRows, pResBlock, keyBufSize, pTaskInfo->id.str); + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 8739371dd9..8018a8dd31 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -265,7 +265,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - toSDatablock(pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + toSDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -307,11 +307,11 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou // pInfo->binfo.rowCellInfoOffset); // } - blockDataEnsureCapacity(pRes, pInfo->binfo.capacity); + blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); while(1) { - toSDatablock(pRes, pInfo->binfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); + toSDatablock(pRes, pOperator->resultInfo.capacity, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset); doFilter(pInfo->pCondition, pRes); bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo); @@ -348,7 +348,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx goto _error; } - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pInfo->groupKeyLen, pTaskInfo->id.str); + initResultSizeInfo(pOperator, 4096); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResultBlock, pInfo->groupKeyLen, pTaskInfo->id.str); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pOperator->name = "GroupbyAggOperator"; From 9d1a9d9510aa7adfb5f1ffa760bad3bbb247f122 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Apr 2022 19:27:41 +0800 Subject: [PATCH 07/31] fix(query): avoid the output result overlap within the project operator buffer. --- source/common/src/tdatablock.c | 1 + source/libs/executor/src/executorimpl.c | 41 +++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index beabc1b6eb..8f10fa58cc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -203,6 +203,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co if (pSource->hasNull) { pColumnInfoData->hasNull = pSource->hasNull; } + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { // Handle the bitmap char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index beeea4f223..0fdb523e64 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1171,31 +1171,43 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.groupId = pSrcBlock->info.groupId; + int32_t numOfRows = 0; + for (int32_t k = 0; k < numOfOutput; ++k) { int32_t outputSlotId = pExpr[k].base.resSchema.slotId; SqlFunctionCtx* pfCtx = &pCtx[k]; if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); - colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); + if (pResult->info.rows > 0) { + colDataMergeCol(pColInfoData, pResult->info.rows, pfCtx->input.pData[0], pfCtx->input.numOfRows); + } else { + colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); + } - pResult->info.rows = pSrcBlock->info.rows; + numOfRows = pfCtx->input.numOfRows; } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) { SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); + + int32_t offset = pResult->info.rows; for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].param.nType), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType); + colDataAppend(pColInfoData, i + offset, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].param.nType), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType); } - pResult->info.rows = pSrcBlock->info.rows; + + numOfRows = pSrcBlock->info.rows; } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_OPERATOR) { SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); taosArrayPush(pBlockList, &pSrcBlock); - SScalarParam dest = {0}; - dest.columnData = taosArrayGet(pResult->pDataBlock, outputSlotId); + SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId); + SColumnInfoData idata = {.info = pResColData->info}; + SScalarParam dest = {.columnData = &idata}; scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - pResult->info.rows = dest.numOfRows; + colDataMergeCol(pResColData, pResult->info.rows, &idata, dest.numOfRows); + + numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { ASSERT(!fmIsAggFunc(pfCtx->functionId)); @@ -1212,28 +1224,33 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); pfCtx->offset = pResult->info.rows; // set the start offset + // set the timestamp(_rowts) output buffer if (taosArrayGetSize(pPseudoList) > 0) { int32_t* outputColIndex = taosArrayGet(pPseudoList, 0); pfCtx->pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput; } - int32_t numOfRows = pfCtx->fpSet.process(pfCtx); - pResult->info.rows += numOfRows; + numOfRows = pfCtx->fpSet.process(pfCtx); } else { SArray* pBlockList = taosArrayInit(4, POINTER_BYTES); taosArrayPush(pBlockList, &pSrcBlock); - SScalarParam dest = {0}; - dest.columnData = taosArrayGet(pResult->pDataBlock, outputSlotId); + SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId); + SColumnInfoData idata = {.info = pResColData->info}; + SScalarParam dest = {.columnData = &idata}; scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); - pResult->info.rows = dest.numOfRows; + colDataMergeCol(pResColData, pResult->info.rows, &idata, dest.numOfRows); + + numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); } } else { ASSERT(0); } } + + pResult->info.rows += numOfRows; } void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, From 68b69be5fafc13c046cefc6de990a248af1c4d34 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Mon, 18 Apr 2022 20:52:10 +0800 Subject: [PATCH 08/31] fix(os): centos pthread spinlock error. --- include/os/osThread.h | 13 +++------ source/os/src/osThread.c | 58 +++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/include/os/osThread.h b/include/os/osThread.h index a145b0da07..18c1b290ab 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,6 +22,10 @@ extern "C" { #endif +#ifndef __USE_XOPEN2K +typedef pthread_mutex_t pthread_spinlock_t; +#endif + typedef pthread_t TdThread; typedef pthread_spinlock_t TdThreadSpinlock; typedef pthread_mutex_t TdThreadMutex; @@ -33,8 +37,6 @@ typedef pthread_rwlockattr_t TdThreadRwlockAttr; typedef pthread_cond_t TdThreadCond; typedef pthread_condattr_t TdThreadCondAttr; typedef pthread_key_t TdThreadKey; -typedef pthread_barrier_t TdThreadBarrier; -typedef pthread_barrierattr_t TdThreadBarrierAttr; #define taosThreadCleanupPush pthread_cleanup_push #define taosThreadCleanupPop pthread_cleanup_pop @@ -156,13 +158,6 @@ int32_t taosThreadAttrSetSchedParam(TdThreadAttr * attr, const struct sched_para int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr * attr, int32_t policy); int32_t taosThreadAttrSetScope(TdThreadAttr * attr, int32_t contentionscope); int32_t taosThreadAttrSetStackSize(TdThreadAttr * attr, size_t stacksize); -int32_t taosThreadBarrierDestroy(TdThreadBarrier * barrier); -int32_t taosThreadBarrierInit(TdThreadBarrier * barrier, const TdThreadBarrierAttr * attr, uint32_t count); -int32_t taosThreadBarrierWait(TdThreadBarrier * barrier); -int32_t taosThreadBarrierAttrDestroy(TdThreadBarrierAttr * attr); -int32_t taosThreadBarrierAttrGetPshared(const TdThreadBarrierAttr * attr, int32_t *pshared); -int32_t taosThreadBarrierAttrInit(TdThreadBarrierAttr * attr); -int32_t taosThreadBarrierAttrSetPshared(TdThreadBarrierAttr * attr, int32_t pshared); int32_t taosThreadCancel(TdThread thread); int32_t taosThreadCondDestroy(TdThreadCond * cond); int32_t taosThreadCondInit(TdThreadCond * cond, const TdThreadCondAttr * attr); diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 436614ad6a..656871fd3c 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -77,34 +77,6 @@ int32_t taosThreadAttrSetStackSize(TdThreadAttr * attr, size_t stacksize) { return pthread_attr_setstacksize(attr, stacksize); } -int32_t taosThreadBarrierDestroy(TdThreadBarrier * barrier) { - return pthread_barrier_destroy(barrier); -} - -int32_t taosThreadBarrierInit(TdThreadBarrier * barrier, const TdThreadBarrierAttr * attr, uint32_t count) { - return pthread_barrier_init(barrier, attr, count); -} - -int32_t taosThreadBarrierWait(TdThreadBarrier * barrier) { - return pthread_barrier_wait(barrier); -} - -int32_t taosThreadBarrierAttrDestroy(TdThreadBarrierAttr * attr) { - return pthread_barrierattr_destroy(attr); -} - -int32_t taosThreadBarrierAttrGetPshared(const TdThreadBarrierAttr * attr, int32_t *pshared) { - return pthread_barrierattr_getpshared(attr, pshared); -} - -int32_t taosThreadBarrierAttrInit(TdThreadBarrierAttr * attr) { - return pthread_barrierattr_init(attr); -} - -int32_t taosThreadBarrierAttrSetPshared(TdThreadBarrierAttr * attr, int32_t pshared) { - return pthread_barrierattr_setpshared(attr, pshared); -} - int32_t taosThreadCancel(TdThread thread) { return pthread_cancel(thread); } @@ -322,23 +294,43 @@ int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { } int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { - return pthread_spin_destroy(lock); +#ifndef __USE_XOPEN2K + return pthread_mutex_destroy((pthread_mutex_t*)lock); +#else + return pthread_spin_destroy((pthread_spinlock_t*)lock); +#endif } int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) { - return pthread_spin_init(lock, pshared); +#ifndef __USE_XOPEN2K + return pthread_mutex_init((pthread_mutex_t*)lock, pshared); +#else + return pthread_spin_init((pthread_spinlock_t*)lock, pshared); +#endif } int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { - return pthread_spin_lock(lock); +#ifndef __USE_XOPEN2K + return pthread_mutex_lock((pthread_mutex_t*)lock); +#else + return pthread_spin_lock((pthread_spinlock_t*)lock); +#endif } int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { - return pthread_spin_trylock(lock); +#ifndef __USE_XOPEN2K + return pthread_mutex_trylock((pthread_mutex_t*)lock); +#else + return pthread_spin_trylock((pthread_spinlock_t*)lock); +#endif } int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock) { - return pthread_spin_unlock(lock); +#ifndef __USE_XOPEN2K + return pthread_mutex_unlock((pthread_mutex_t*)lock); +#else + return pthread_spin_unlock((pthread_spinlock_t*)lock); +#endif } void taosThreadTestCancel(void) { From 10f99917c94f8d8ac76088023df1cef7e708b13d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 Apr 2022 21:14:32 +0800 Subject: [PATCH 09/31] style: format shell --- tools/shell/src/shellCommand.c | 11 ++- tools/shell/src/shellEngine.c | 58 ++++++++------- tools/shell/src/shellMain.c | 125 +++++++++++++++------------------ 3 files changed, 93 insertions(+), 101 deletions(-) diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 7cbadfaf5b..546b19f83c 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -15,9 +15,9 @@ #define __USE_XOPEN +#include "shellCommand.h" #include "os.h" #include "shell.h" -#include "shellCommand.h" #include @@ -48,7 +48,7 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { while (--pos >= 0) { *size += 1; - if (str[pos] > 0 || countPrefixOnes((unsigned char )str[pos]) > 1) break; + if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break; } int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); @@ -106,8 +106,7 @@ void clearLineBefore(Command *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - memmove(cmd->command, cmd->command + cmd->cursorOffset, - cmd->commandSize - cmd->cursorOffset); + memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); cmd->commandSize -= cmd->cursorOffset; cmd->cursorOffset = 0; cmd->screenOffset = 0; @@ -235,8 +234,8 @@ int isReadyGo(Command *cmd) { sprintf(total, "%s%s", cmd->buffer, cmd->command); char *reg_str = - "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" - "\\s*clear\\s*$)"; + "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" + "\\s*clear\\s*$)"; if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { taosMemoryFree(total); return 1; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 548d2169b1..36d2866fb5 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -23,40 +23,43 @@ #include "shellCommand.h" #include "taosdef.h" #include "taoserror.h" +#include "tconfig.h" #include "tglobal.h" #include "ttypes.h" #include "tutil.h" -#include "tconfig.h" #include #include /**************** Global variables ****************/ #ifdef _TD_POWER_ -char CLIENT_VERSION[] = "Welcome to the PowerDB shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "power> "; +char CLIENT_VERSION[] = + "Welcome to the PowerDB shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "power> "; -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 7; +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 7; #elif (_TD_TQ_ == true) -char CLIENT_VERSION[] = "Welcome to the TQ shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "tq> "; +char CLIENT_VERSION[] = + "Welcome to the TQ shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "tq> "; -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 4; +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 4; #else -char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "taos> "; +char CLIENT_VERSION[] = + "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "taos> "; -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 6; +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 6; #endif -int64_t result = 0; -SShellHistory history; +int64_t result = 0; +SShellHistory history; #define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 extern int32_t tsMaxBinaryDisplayWidth; @@ -339,12 +342,12 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { atomic_store_64(&result, 0); freeResultWithRid(oresult); - taos_free_result(pSql); - + taos_free_result(pSql); + return; } - TAOS_FIELD* pFields = taos_fetch_fields(pSql); + TAOS_FIELD *pFields = taos_fetch_fields(pSql); if (pFields != NULL) { // select and show kinds of commands int error_no = 0; @@ -361,7 +364,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } else { printf("Query interrupted (%s), %d row(s) in set (%.6fs)\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); } - taos_free_result(pSql); + taos_free_result(pSql); } else { int num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); @@ -521,7 +524,8 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { } // FILE *fp = fopen(full_path.we_wordv[0], "w"); - TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + TdFilePtr pFile = + taosOpenFile(full_path.we_wordv[0], TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); @@ -901,8 +905,8 @@ void read_history() { memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE); history.hstart = 0; history.hend = 0; - char *line = NULL; - int read_size = 0; + char *line = NULL; + int read_size = 0; char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); @@ -929,7 +933,7 @@ void read_history() { } } - if(line != NULL) taosMemoryFree(line); + if (line != NULL) taosMemoryFree(line); taosCloseFile(&pFile); } @@ -1026,7 +1030,7 @@ void source_file(TAOS *con, char *fptr) { } taosMemoryFree(cmd); - if(line != NULL) taosMemoryFree(line); + if (line != NULL) taosMemoryFree(line); wordfree(&full_path); taosCloseFile(&pFile); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 8ca8142eca..0d397eb80b 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -14,45 +14,38 @@ */ #define __USE_XOPEN -#include "os.h" -#include "shell.h" -#include "tglobal.h" -#include "tconfig.h" #include "shellCommand.h" -#include "tbase64.h" +#include "tglobal.h" #include "tlog.h" -#include "version.h" -#include #include #include +#include #define OPT_ABORT 1 /* abort */ - int indicator = 1; -void insertChar(Command *cmd, char *c, int size); -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, - int32_t pkgNum, char *pkgType); +void insertChar(Command *cmd, char *c, int size); +void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); const char *argp_program_version = version; const char *argp_program_bug_address = ""; static char doc[] = ""; static char args_doc[] = ""; -TdThread pid; +TdThread pid; static tsem_t cancelSem; static struct argp_option options[] = { {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, - {"password", 'p', 0, 0, "The password to use when connecting to the server."}, + {"password", 'p', NULL, 0, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, - {"dump-config", 'C', 0, 0, "Dump configuration."}, + {"dump-config",'C', NULL, 0, "Dump configuration."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, - {"raw-time", 'r', 0, 0, "Output time as uint64_t."}, + {"raw-time", 'r', NULL, 0, "Output time as uint64_t."}, {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, {"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."}, {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, @@ -70,7 +63,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we know is a pointer to our arguments structure. */ SShellArguments *arguments = state->input; - wordexp_t full_path; + wordexp_t full_path; switch (key) { case 'h': @@ -80,7 +73,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'P': if (arg) { - arguments->port = atoi(arg); + arguments->port = atoi(arg); } else { fprintf(stderr, "Invalid port\n"); return -1; @@ -182,35 +175,33 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Our argp parser. */ static struct argp argp = {options, parse_opt, args_doc, doc}; -char LINUXCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; +char LINUXCLIENT_VERSION[] = + "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; char g_password[SHELL_MAX_PASSWORD_LEN]; -static void parse_args( - int argc, char *argv[], SShellArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strncmp(argv[i], "-p", 2) == 0) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%20s", g_password) > 1) { - fprintf(stderr, "password reading error\n"); - } - taosSetConsoleEcho(true); - if (EOF == getchar()) { - fprintf(stderr, "getchar() return EOF\n"); - } - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - strcpy(argv[i], "-p"); - } - arguments->password = g_password; - arguments->is_use_passwd = true; +static void parse_args(int argc, char *argv[], SShellArguments *arguments) { + for (int i = 1; i < argc; i++) { + if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { + printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); + if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) { + printf("Enter password: "); + taosSetConsoleEcho(false); + if (scanf("%20s", g_password) > 1) { + fprintf(stderr, "password reading error\n"); } + taosSetConsoleEcho(true); + if (EOF == getchar()) { + fprintf(stderr, "getchar() return EOF\n"); + } + } else { + tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); + strcpy(argv[i], "-p"); + } + arguments->password = g_password; + arguments->is_use_passwd = true; } + } } void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { @@ -225,20 +216,20 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { argp_parse(&argp, argc, argv, 0, 0, arguments); if (arguments->abort) { - #ifndef _ALPINE - #if 0 +#ifndef _ALPINE +#if 0 error(10, 0, "ABORTED"); - #endif - #else - abort(); - #endif +#endif +#else + abort(); +#endif } } int32_t shellReadCommand(TAOS *con, char *command) { unsigned hist_counter = history.hend; - char utf8_array[10] = "\0"; - Command cmd; + char utf8_array[10] = "\0"; + Command cmd; memset(&cmd, 0, sizeof(cmd)); cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); @@ -247,7 +238,7 @@ int32_t shellReadCommand(TAOS *con, char *command) { // Read input. char c; while (1) { - c = (char)getchar(); // getchar() return an 'int' value + c = (char)getchar(); // getchar() return an 'int' value if (c == EOF) { return c; @@ -406,13 +397,13 @@ void *shellLoopQuery(void *arg) { taosThreadCleanupPush(cleanup_handler, NULL); char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL){ + if (command == NULL) { uError("failed to malloc command"); return NULL; } int32_t err = 0; - + do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); @@ -423,12 +414,12 @@ void *shellLoopQuery(void *arg) { } resetTerminalMode(); } while (shellRunCommand(con, command) == 0); - + taosMemoryFreeClear(command); exitShell(); taosThreadCleanupPop(1); - + return NULL; } @@ -437,7 +428,7 @@ void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "% void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); + // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); w.ws_col = 120; w.ws_row = 30; } @@ -458,13 +449,13 @@ void clearScreen(int ecmd_pos, int cursor_pos) { void showOnScreen(Command *cmd) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device\n"); + // fprintf(stderr, "No stream device\n"); w.ws_col = 120; w.ws_row = 30; } TdWchar wc; - int size = 0; + int size = 0; // Print out the command. char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); @@ -531,13 +522,11 @@ void showOnScreen(Command *cmd) { void cleanup_handler(void *arg) { resetTerminalMode(); } void exitShell() { - /*int32_t ret =*/ resetTerminalMode(); + /*int32_t ret =*/resetTerminalMode(); taos_cleanup(); exit(EXIT_SUCCESS); } -void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { - tsem_post(&cancelSem); -} +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } void *cancelHandler(void *arg) { setThreadName("cancelHandler"); @@ -554,7 +543,7 @@ void *cancelHandler(void *arg) { SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); taos_stop_query(pSql); taosReleaseRef(tscObjRef, rid); -#endif +#endif #else resetTerminalMode(); printf("\nReceive ctrl+c or other signal, quit shell.\n"); @@ -640,11 +629,11 @@ int main(int argc, char *argv[]) { con = taos_connect_auth(args.host, args.user, args.auth, args.database, args.port); } -/* if (taos_init()) { - printf("Failed to init taos"); - exit(EXIT_FAILURE); - } - */ + // if (taos_init()) { + // printf("Failed to init taos"); + // exit(EXIT_FAILURE); + // } + taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); taos_close(con); exit(0); From 9329f5f058f069b2058e0484277f9f637cce1239 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 18 Apr 2022 13:23:22 +0000 Subject: [PATCH 10/31] refact vnode --- include/common/tmsg.h | 29 ++--- source/common/src/tmsg.c | 16 +-- source/dnode/mgmt/test/vnode/CMakeLists.txt | 20 ++-- source/dnode/mnode/impl/src/mndStb.c | 72 ++++--------- source/dnode/vnode/src/meta/metaTDBImpl.c | 26 ++--- source/libs/parser/src/parTranslater.c | 111 +++++++++++--------- 6 files changed, 122 insertions(+), 152 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 968db0c2ad..8c30f7d280 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -253,22 +253,16 @@ typedef struct { SSubmitRspBlock failedBlocks[]; } SSubmitRsp; +#define SCHEMA_SMA_ON 0x1 +#define SCHEMA_IDX_ON 0x2 typedef struct SSchema { int8_t type; - int8_t index; // default is 0, not index created + int8_t flags; col_id_t colId; int32_t bytes; char name[TSDB_COL_NAME_LEN]; } SSchema; -typedef struct { - int8_t type; - int8_t sma; // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I - col_id_t colId; - int32_t bytes; - char name[TSDB_COL_NAME_LEN]; -} SSchemaEx; - #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_SMA(s) ((s)->sma) #define SSCHMEA_COLID(s) ((s)->colId) @@ -1454,7 +1448,7 @@ typedef struct SVCreateTbReq { tb_uid_t suid; col_id_t nCols; col_id_t nBSmaCols; - SSchemaEx* pSchema; + SSchema* pSchema; col_id_t nTagCols; SSchema* pTagSchema; SRSmaParam* pRSmaParam; @@ -1466,7 +1460,7 @@ typedef struct SVCreateTbReq { struct { col_id_t nCols; col_id_t nBSmaCols; - SSchemaEx* pSchema; + SSchema* pSchema; SRSmaParam* pRSmaParam; } ntbCfg; }; @@ -2031,16 +2025,13 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq) typedef struct { uint32_t nCols; - union { - SSchema* pSchema; - SSchemaEx* pSchemaEx; - }; + SSchema* pSchema; } SSchemaWrapper; static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI8(buf, pSchema->index); + tlen += taosEncodeFixedI8(buf, pSchema->flags); tlen += taosEncodeFixedI32(buf, pSchema->bytes); tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeString(buf, pSchema->name); @@ -2049,7 +2040,7 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI8(buf, &pSchema->index); + buf = taosDecodeFixedI8(buf, &pSchema->flags); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeStringTo(buf, pSchema->name); @@ -2058,7 +2049,7 @@ static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; - if (tEncodeI8(pEncoder, pSchema->index) < 0) return -1; + if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1; if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; @@ -2067,7 +2058,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; - if (tDecodeI8(pDecoder, &pSchema->index) < 0) return -1; + if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1; if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a21fc81bfa..6df2341af4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -411,7 +411,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols); for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma); + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].flags); tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name); @@ -419,7 +419,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols); for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type); - tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].index); + tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].flags); tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes); tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name); @@ -443,7 +443,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols); for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type); - tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma); + tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].flags); tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId); tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes); tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name); @@ -478,10 +478,10 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); - pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx)); + pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma)); + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].flags)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes)); buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name); @@ -490,7 +490,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); - buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].index)); + buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].flags)); buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); @@ -520,10 +520,10 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { case TD_NORMAL_TABLE: buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols); buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); - pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx)); + pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema)); for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); - buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma); + buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].flags); buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes); buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name); diff --git a/source/dnode/mgmt/test/vnode/CMakeLists.txt b/source/dnode/mgmt/test/vnode/CMakeLists.txt index 34402286aa..c399474277 100644 --- a/source/dnode/mgmt/test/vnode/CMakeLists.txt +++ b/source/dnode/mgmt/test/vnode/CMakeLists.txt @@ -1,11 +1,11 @@ -aux_source_directory(. DND_VNODE_TEST_SRC) -add_executable(dvnodeTest ${DND_VNODE_TEST_SRC}) -target_link_libraries( - dvnodeTest - PUBLIC sut -) +# aux_source_directory(. DND_VNODE_TEST_SRC) +# add_executable(dvnodeTest ${DND_VNODE_TEST_SRC}) +# target_link_libraries( +# dvnodeTest +# PUBLIC sut +# ) -add_test( - NAME dvnodeTest - COMMAND dvnodeTest -) +# add_test( +# NAME dvnodeTest +# COMMAND dvnodeTest +# ) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 777141c5d2..2aa20df2e6 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -18,8 +18,8 @@ #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" -#include "mndPerfSchema.h" #include "mndMnode.h" +#include "mndPerfSchema.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -40,7 +40,7 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp); static int32_t mndProcessVAlterStbRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropStbRsp(SNodeMsg *pRsp); static int32_t mndProcessTableMetaReq(SNodeMsg *pReq); -static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); int32_t mndInitStb(SMnode *pMnode) { @@ -333,9 +333,9 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { } static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSchema) { - if (*(col_id_t *)colId < ((SSchemaEx *)pSchema)->colId) { + if (*(col_id_t *)colId < ((SSchema *)pSchema)->colId) { return -1; - } else if (*(col_id_t *)colId > ((SSchemaEx *)pSchema)->colId) { + } else if (*(col_id_t *)colId > ((SSchema *)pSchema)->colId) { return 1; } return 0; @@ -360,49 +360,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.stbCfg.nTagCols = pStb->numOfTags; req.stbCfg.pTagSchema = pStb->pTags; req.stbCfg.nBSmaCols = pStb->numOfSmas; - req.stbCfg.pSchema = (SSchemaEx *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaEx)); + req.stbCfg.pSchema = (SSchema *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); if (req.stbCfg.pSchema == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - int bSmaStat = 0; // no column has bsma - if (pStb->numOfSmas == pStb->numOfColumns) { // assume pColumns > 0 - bSmaStat = 1; // all columns have bsma - } else if (pStb->numOfSmas != 0) { - bSmaStat = 2; // partial columns have bsma - TASSERT(pStb->pSmas != NULL); // TODO: remove the assert - } - - for (int32_t i = 0; i < req.stbCfg.nCols; ++i) { - SSchemaEx *pSchemaEx = req.stbCfg.pSchema + i; - SSchema *pSchema = pStb->pColumns + i; - pSchemaEx->type = pSchema->type; - pSchemaEx->sma = (bSmaStat == 1) ? TSDB_BSMA_TYPE_LATEST : TSDB_BSMA_TYPE_NONE; - pSchemaEx->colId = pSchema->colId; - pSchemaEx->bytes = pSchema->bytes; - memcpy(pSchemaEx->name, pSchema->name, TSDB_COL_NAME_LEN); - } - - if (bSmaStat == 2) { - if (pStb->pSmas == NULL) { - mError("stb:%s, sma options is empty", pStb->name); - taosMemoryFreeClear(req.stbCfg.pSchema); - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - return NULL; - } - for (int32_t i = 0; i < pStb->numOfSmas; ++i) { - SSchema *pSmaSchema = pStb->pSmas + i; - SSchemaEx *pColSchema = taosbsearch(&pSmaSchema->colId, req.stbCfg.pSchema, req.stbCfg.nCols, sizeof(SSchemaEx), - schemaExColIdCompare, TD_EQ); - if (pColSchema == NULL) { - terrno = TSDB_CODE_MND_INVALID_STB_OPTION; - taosMemoryFreeClear(req.stbCfg.pSchema); - mError("stb:%s, sma col:%s not found in columns", pStb->name, pSmaSchema->name); - return NULL; - } - pColSchema->sma = TSDB_BSMA_TYPE_LATEST; - } + memcpy(req.stbCfg.pSchema, pStb->pColumns, sizeof(SSchema) * pStb->numOfColumns); + for (int i = 0; i < pStb->numOfColumns; i++) { + req.stbCfg.pSchema[i].flags = SCHEMA_SMA_ON; } SRSmaParam *pRSmaParam = NULL; @@ -428,9 +394,9 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod; } req.stbCfg.pRSmaParam = pRSmaParam; - } + } - int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { if (pRSmaParam) { @@ -1644,14 +1610,14 @@ static void mndExtractTableName(char *tableId, char *name) { } } -static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStbObj *pStb = NULL; int32_t cols = 0; - SDbObj* pDb = NULL; + SDbObj *pDb = NULL; if (strlen(pShow->db) > 0) { pDb = mndAcquireDb(pMnode, pShow->db); if (pDb == NULL) return terrno; @@ -1669,20 +1635,20 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo cols = 0; SName name = {0}; - char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; mndExtractTableName(pStb->name, &stbName[VARSTR_HEADER_SIZE]); varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) stbName, false); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); - char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&name, pStb->db, T_NAME_ACCT|T_NAME_DB); + char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); tNameGetDbName(&name, varDataVal(db)); varDataSetLen(db, strlen(varDataVal(db))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*) db, false); + colDataAppend(pColInfo, numOfRows, (const char *)db, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); @@ -1696,7 +1662,7 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables - char* p = taosMemoryMalloc(pStb->commentLen + VARSTR_HEADER_SIZE); // check malloc failures + char *p = taosMemoryMalloc(pStb->commentLen + VARSTR_HEADER_SIZE); // check malloc failures if (p != NULL) { if (pStb->commentLen != 0) { STR_TO_VARSTR(p, pStb->comment); diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 9b9f54b5ba..9fd11222bf 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -299,10 +299,10 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { if (pTbCfg->type == META_SUPER_TABLE) { schemaWrapper.nCols = pTbCfg->stbCfg.nCols; - schemaWrapper.pSchemaEx = pTbCfg->stbCfg.pSchema; + schemaWrapper.pSchema = pTbCfg->stbCfg.pSchema; } else { schemaWrapper.nCols = pTbCfg->ntbCfg.nCols; - schemaWrapper.pSchemaEx = pTbCfg->ntbCfg.pSchema; + schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema; } pVal = pBuf = buf; metaEncodeSchemaEx(&pBuf, &schemaWrapper); @@ -464,7 +464,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { tb_uid_t quid; SSchemaWrapper *pSW; STSchemaBuilder sb; - SSchemaEx *pSchema; + SSchema *pSchema; STSchema *pTSchema; STbCfg *pTbCfg; @@ -482,8 +482,8 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { tdInitTSchemaBuilder(&sb, 0); for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchemaEx + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->sma, pSchema->colId, pSchema->bytes); + pSchema = pSW->pSchema + i; + tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); } pTSchema = tdGetSchemaFromBuilder(&sb); tdDestroyTSchemaBuilder(&sb); @@ -939,7 +939,7 @@ static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { for (int i = 0; i < pSW->nCols; i++) { pSchema = pSW->pSchema + i; tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI8(buf, pSchema->index); + tlen += taosEncodeFixedI8(buf, pSchema->flags); tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeFixedI32(buf, pSchema->bytes); tlen += taosEncodeString(buf, pSchema->name); @@ -966,14 +966,14 @@ static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { } static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) { - int tlen = 0; - SSchemaEx *pSchema; + int tlen = 0; + SSchema *pSchema; tlen += taosEncodeFixedU32(buf, pSW->nCols); for (int i = 0; i < pSW->nCols; ++i) { - pSchema = pSW->pSchemaEx + i; + pSchema = pSW->pSchema + i; tlen += taosEncodeFixedI8(buf, pSchema->type); - tlen += taosEncodeFixedI8(buf, pSchema->sma); + tlen += taosEncodeFixedI8(buf, pSchema->flags); tlen += taosEncodeFixedI16(buf, pSchema->colId); tlen += taosEncodeFixedI32(buf, pSchema->bytes); tlen += taosEncodeString(buf, pSchema->name); @@ -985,11 +985,11 @@ static int metaEncodeSchemaEx(void **buf, SSchemaWrapper *pSW) { static void *metaDecodeSchemaEx(void *buf, SSchemaWrapper *pSW, bool isGetEx) { buf = taosDecodeFixedU32(buf, &pSW->nCols); if (isGetEx) { - pSW->pSchemaEx = (SSchemaEx *)taosMemoryMalloc(sizeof(SSchemaEx) * pSW->nCols); + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); for (int i = 0; i < pSW->nCols; i++) { - SSchemaEx *pSchema = pSW->pSchemaEx + i; + SSchema *pSchema = pSW->pSchema + i; buf = taosDecodeFixedI8(buf, &pSchema->type); - buf = taosDecodeFixedI8(buf, &pSchema->sma); + buf = taosDecodeFixedI8(buf, &pSchema->flags); buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeStringTo(buf, pSchema->name); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 45e8cd3acc..8b5632a427 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -495,7 +495,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_INTEGER_TYPE(rdt.type)) || (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_INTEGER_TYPE(ldt.type)) || (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_BOOL == rdt.type) || - (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type) ) { + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && TSDB_DATA_TYPE_BOOL == ldt.type)) { pOp->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; } else { @@ -809,7 +809,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) { size_t nums = taosArrayGetSize(pTables); for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols); + int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -835,7 +835,8 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { return NULL; } pFunc->pParameterList = nodesMakeList(); - if (NULL == pFunc->pParameterList || TSDB_CODE_SUCCESS != nodesListStrictAppend(pFunc->pParameterList, nodesCloneNode(pExpr))) { + if (NULL == pFunc->pParameterList || + TSDB_CODE_SUCCESS != nodesListStrictAppend(pFunc->pParameterList, nodesCloneNode(pExpr))) { nodesDestroyNode(pFunc); return NULL; } @@ -844,7 +845,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) { pFunc->funcId = pSrcFunc->funcId; pFunc->funcType = pSrcFunc->funcType; strcpy(pFunc->functionName, pSrcFunc->functionName); - char buf[TSDB_FUNC_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + char buf[TSDB_FUNC_NAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = 0; if (QUERY_NODE_COLUMN == nodeType(pExpr)) { SColumnNode* pCol = (SColumnNode*)pExpr; @@ -872,7 +873,7 @@ static int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STabl static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SNodeList** pOutput) { STableNode* pTable = NULL; - int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); + int32_t code = findTable(pCxt, pCol->tableAlias, &pTable); if (TSDB_CODE_SUCCESS == code && NULL == *pOutput) { *pOutput = nodesMakeList(); if (NULL == *pOutput) { @@ -886,18 +887,20 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SN } static bool isStar(SNode* pNode) { - return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) && (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); + return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' == ((SColumnNode*)pNode)->tableAlias[0]) && + (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); } static bool isTableStar(SNode* pNode) { - return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) && (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); + return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) && + (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); } static int32_t createMultiResFuncsParas(STranslateContext* pCxt, SNodeList* pSrcParas, SNodeList** pOutput) { int32_t code = TSDB_CODE_SUCCESS; SNodeList* pExprs = NULL; - SNode* pPara = NULL; + SNode* pPara = NULL; FOREACH(pPara, pSrcParas) { if (isStar(pPara)) { code = createAllColumns(pCxt, &pExprs); @@ -929,7 +932,7 @@ static int32_t createMultiResFuncs(SFunctionNode* pSrcFunc, SNodeList* pExprs, S } int32_t code = TSDB_CODE_SUCCESS; - SNode* pExpr = NULL; + SNode* pExpr = NULL; FOREACH(pExpr, pExprs) { code = nodesListStrictAppend(pFuncs, createMultiResFunc(pSrcFunc, (SExprNode*)pExpr)); if (TSDB_CODE_SUCCESS != code) { @@ -948,7 +951,7 @@ static int32_t createMultiResFuncs(SFunctionNode* pSrcFunc, SNodeList* pExprs, S static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNode* pSrcFunc, SNodeList** pOutput) { SNodeList* pExprs = NULL; - int32_t code = createMultiResFuncsParas(pCxt, pSrcFunc->pParameterList, &pExprs); + int32_t code = createMultiResFuncsParas(pCxt, pSrcFunc->pParameterList, &pExprs); if (TSDB_CODE_SUCCESS == code) { code = createMultiResFuncs(pSrcFunc, pExprs, pOutput); } @@ -970,8 +973,8 @@ static bool isCountStar(SNode* pNode) { static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); } @@ -1125,9 +1128,7 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return translateExprList(pCxt, pSelect->pGroupByList); } -static bool isValTimeUnit(char unit) { - return ('n' == unit || 'y' == unit); -} +static bool isValTimeUnit(char unit) { return ('n' == unit || 'y' == unit); } static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); @@ -1155,8 +1156,8 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = isValTimeUnit(pInter->unit); - if (pInter->datum.i <= 0 || + bool valInter = isValTimeUnit(pInter->unit); + if (pInter->datum.i <= 0 || (!valInter && convertTimePrecision(pInter->datum.i, precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime); } @@ -1167,18 +1168,19 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE); } if (pInter->unit == 'n' && pOffset->unit == 'y') { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } bool fixed = !isValTimeUnit(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || - (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { + (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= + getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); } } if (NULL != pInterval->pSliding) { const static int32_t INTERVAL_SLIDING_FACTOR = 100; - + SValueNode* pSliding = (SValueNode*)pInterval->pSliding; if (pInter->unit == 'n' || pInter->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); @@ -1288,14 +1290,14 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { static int64_t getUnitPerMinute(uint8_t precision) { switch (precision) { - case TSDB_TIME_PRECISION_MILLI: - return MILLISECOND_PER_MINUTE; - case TSDB_TIME_PRECISION_MICRO: - return MILLISECOND_PER_MINUTE * 1000L; - case TSDB_TIME_PRECISION_NANO: - return NANOSECOND_PER_MINUTE; - default: - break; + case TSDB_TIME_PRECISION_MILLI: + return MILLISECOND_PER_MINUTE; + case TSDB_TIME_PRECISION_MICRO: + return MILLISECOND_PER_MINUTE * 1000L; + case TSDB_TIME_PRECISION_NANO: + return NANOSECOND_PER_MINUTE; + default: + break; } return MILLISECOND_PER_MINUTE; } @@ -1364,7 +1366,8 @@ static int32_t checkRangeOption(STranslateContext* pCxt, const char* pName, SVal if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } - if (pVal->isDuration && (TIME_UNIT_MINUTE != pVal->unit && TIME_UNIT_HOUR != pVal->unit && TIME_UNIT_DAY != pVal->unit)) { + if (pVal->isDuration && + (TIME_UNIT_MINUTE != pVal->unit && TIME_UNIT_HOUR != pVal->unit && TIME_UNIT_DAY != pVal->unit)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_OPTION_UNIT, pName, pVal->unit); } int64_t val = getBigintFromValueNode(pVal); @@ -1459,9 +1462,12 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { SValueNode* pKeep0 = (SValueNode*)nodesListGetNode(pKeep, 0); SValueNode* pKeep1 = (SValueNode*)nodesListGetNode(pKeep, 1); SValueNode* pKeep2 = (SValueNode*)nodesListGetNode(pKeep, 2); - if ((pKeep0->isDuration && (TIME_UNIT_MINUTE != pKeep0->unit && TIME_UNIT_HOUR != pKeep0->unit && TIME_UNIT_DAY != pKeep0->unit)) || - (pKeep1->isDuration && (TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || - (pKeep2->isDuration && (TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { + if ((pKeep0->isDuration && + (TIME_UNIT_MINUTE != pKeep0->unit && TIME_UNIT_HOUR != pKeep0->unit && TIME_UNIT_DAY != pKeep0->unit)) || + (pKeep1->isDuration && + (TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || + (pKeep2->isDuration && + (TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit); } @@ -1518,7 +1524,8 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, SDatabaseOptions* p code = checkRangeOption(pCxt, "compression", pOptions->pCompressionLevel, TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); } if (TSDB_CODE_SUCCESS == code) { - code = checkRangeOption(pCxt, "daysPerFile", pOptions->pDaysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + code = + checkRangeOption(pCxt, "daysPerFile", pOptions->pDaysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); } if (TSDB_CODE_SUCCESS == code) { code = checkRangeOption(pCxt, "fsyncPeriod", pOptions->pFsyncPeriod, TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); @@ -1571,7 +1578,7 @@ static int32_t checkCreateDatabase(STranslateContext* pCxt, SCreateDatabaseStmt* return checkDatabaseOptions(pCxt, pStmt->pOptions); } -typedef int32_t (*FSerializeFunc)(void *pBuf, int32_t bufLen, void *pReq); +typedef int32_t (*FSerializeFunc)(void* pBuf, int32_t bufLen, void* pReq); static int32_t buildCmdMsg(STranslateContext* pCxt, int16_t msgType, FSerializeFunc func, void* pReq) { pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); @@ -1607,7 +1614,7 @@ static int32_t translateCreateDatabase(STranslateContext* pCxt, SCreateDatabaseS static int32_t translateDropDatabase(STranslateContext* pCxt, SDropDatabaseStmt* pStmt) { SDropDbReq dropReq = {0}; - SName name = {0}; + SName name = {0}; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); tNameGetFullDbName(&name, dropReq.db); dropReq.ignoreNotExists = pStmt->ignoreNotExists; @@ -1821,7 +1828,7 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt STableMeta* pTableMeta = NULL; SName tableName; int32_t code = getTableMetaImpl( - pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); + pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); if ((TSDB_CODE_TDB_INVALID_TABLE_ID == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { return TSDB_CODE_SUCCESS; } @@ -2107,7 +2114,7 @@ static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt } SMCreateSmaReq createSmaReq = {0}; - int32_t code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); + int32_t code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_SMA, (FSerializeFunc)tSerializeSMCreateSmaReq, &createSmaReq); } @@ -2120,7 +2127,7 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* } static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { SMCreateFullTextReq createFTReq = {0}; - int32_t code = buildCreateFullTextReq(pCxt, pStmt, &createFTReq); + int32_t code = buildCreateFullTextReq(pCxt, pStmt, &createFTReq); if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSMCreateFullTextReq, &createFTReq); } @@ -2176,7 +2183,8 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) { static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) { SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId}; - return buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &createReq); + return buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), + (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &createReq); } static int16_t getDropComponentNodeMsgType(ENodeType type) { @@ -2197,7 +2205,8 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) { static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) { SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId}; - return buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq); + return buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), + (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq); } static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { @@ -2301,11 +2310,13 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* } if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pOptions->pWatermark) { - code = (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark)) ? pCxt->errCode : TSDB_CODE_SUCCESS; + code = (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pWatermark)) ? pCxt->errCode + : TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code) { createReq.triggerType = pStmt->pOptions->triggerType; - createReq.watermark = (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); + createReq.watermark = + (NULL != pStmt->pOptions->pWatermark ? ((SValueNode*)pStmt->pOptions->pWatermark)->datum.i : 0); } if (TSDB_CODE_SUCCESS == code) { @@ -2457,9 +2468,7 @@ static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* nu return TSDB_CODE_SUCCESS; } -static int8_t extractResultTsPrecision(const SSelectStmt* pSelect) { - return pSelect->precision; -} +static int8_t extractResultTsPrecision(const SSelectStmt* pSelect) { return pSelect->precision; } static int32_t extractExplainResultSchema(int32_t* numOfCols, SSchema** pSchema) { *numOfCols = 1; @@ -2722,11 +2731,15 @@ typedef struct SVgroupTablesBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupTablesBatch; -static void toSchemaEx(const SColumnDefNode* pCol, col_id_t colId, SSchemaEx* pSchema) { +static void toSchemaEx(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { + int8_t flags = 0; + if (pCol->sma) { + flags |= SCHEMA_SMA_ON; + } pSchema->colId = colId; pSchema->type = pCol->dataType.type; pSchema->bytes = calcTypeBytes(pCol->dataType); - pSchema->sma = pCol->sma ? TSDB_BSMA_TYPE_LATEST : TSDB_BSMA_TYPE_NONE; + pSchema->flags = flags; strcpy(pSchema->name, pCol->colName); } @@ -2771,7 +2784,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* req.dbFName = strdup(dbFName); req.name = strdup(pStmt->tableName); req.ntbCfg.nCols = LIST_LENGTH(pStmt->pCols); - req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchemaEx)); + req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntbCfg.pSchema) { destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; @@ -3176,7 +3189,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - pQuery->precision = extractResultTsPrecision((SSelectStmt*) pQuery->pRoot); + pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); } if (NULL != pCxt->pDbs) { From c15ec3651348dae5fd7e7afd7e00d5c5712f8877 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 18 Apr 2022 21:33:59 +0800 Subject: [PATCH 11/31] fix(rpc): avoid partial memleak --- source/libs/transport/src/transCli.c | 26 ++++------ source/libs/transport/src/transComm.c | 2 +- source/libs/transport/src/transSrv.c | 74 +++++++++++++++++++++------ 3 files changed, 68 insertions(+), 34 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c655c0bfc9..9dc12a3dec 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -212,10 +212,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd); } \ } while (0) -#define CONN_NO_PERSIST_BY_APP(conn) \ - (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) -#define CONN_RELEASE_BY_SERVER(conn) \ - (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_NO_PERSIST_BY_APP(conn) (((conn)->status == ConnNormal || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) +#define CONN_RELEASE_BY_SERVER(conn) (((conn)->status == ConnRelease || (conn)->status == ConnInPool) && T_REF_VAL_GET(conn) == 1) #define REQUEST_NO_RESP(msg) ((msg)->noResp == 1) #define REQUEST_PERSIS_HANDLE(msg) ((msg)->persistHandle == 1) @@ -290,9 +288,8 @@ void cliHandleResp(SCliConn* conn) { tDebug("%s cli conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, - TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), - taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); + tDebug("%s cli conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pTransInst->label, conn, TMSG_INFO(pHead->msgType), + taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); conn->secured = pHead->secured; @@ -358,12 +355,10 @@ void cliHandleExcept(SCliConn* pConn) { if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); - tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, - TMSG_INFO(transMsg.msgType)); + tDebug("%s cli conn %p construct ahandle %p by %s", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle, TMSG_INFO(transMsg.msgType)); if (transMsg.ahandle == NULL) { transMsg.ahandle = transCtxDumpBrokenlinkVal(&pConn->ctx, (int32_t*)&(transMsg.msgType)); - tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, - transMsg.ahandle); + tDebug("%s cli conn %p construct ahandle %p due to brokenlink", CONN_GET_INST_LABEL(pConn), pConn, transMsg.ahandle); } } else { transMsg.ahandle = pCtx ? pCtx->ahandle : NULL; @@ -546,6 +541,7 @@ static void cliDestroy(uv_handle_t* handle) { transCtxCleanup(&conn->ctx); transQueueDestroy(&conn->cliMsgs); tTrace("%s cli conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); + transDestroyBuffer(&conn->readBuf); taosMemoryFree(conn); } static bool cliHandleNoResp(SCliConn* conn) { @@ -635,9 +631,8 @@ void cliSend(SCliConn* pConn) { pHead->release = REQUEST_RELEASE_HANDLE(pCliMsg) ? 1 : 0; uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, - TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); + tDebug("%s cli conn %p %s is send to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -675,10 +670,9 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); - uv_timer_stop(&pThrd->timer); - pThrd->quit = true; + uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ef9cccccb7..03b97b6fa1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -195,7 +195,7 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) void transDestroyAsyncPool(SAsyncPool* pool) { for (int i = 0; i < pool->nAsync; i++) { uv_async_t* async = &(pool->asyncs[i]); - + uv_close((uv_handle_t*)async, NULL); SAsyncItem* item = async->data; taosThreadMutexDestroy(&item->mtx); taosMemoryFree(item); diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c5a74d4840..50ce0bddcd 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -126,6 +126,11 @@ static void uvWorkerAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle); static void uvShutDownCb(uv_shutdown_t* req, int status); +static void uvFreeCb(uv_handle_t* handle) { + // + taosMemoryFree(handle); +} + static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSrvMsg* msg); @@ -141,8 +146,7 @@ static void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleResp(SSrvMsg* msg, SWorkThrdObj* thrd); static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd); -static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, - uvHandleRegister}; +static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, uvHandleRegister}; static void uvDestroyConn(uv_handle_t* handle); @@ -205,13 +209,12 @@ static void uvHandleReq(SSrvConn* pConn) { } if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), - ntohs(pConn->locaddr.sin_port), transMsg.contLen); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen); } else { - tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, - TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), - taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), transMsg.contLen, pHead->noResp); + tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d ", pConn, TMSG_INFO(transMsg.msgType), + taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port), + transMsg.contLen, pHead->noResp); // no ref here } @@ -318,6 +321,8 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } else { tError("fail to dispatch conn to work thread"); } + uv_close((uv_handle_t*)req->data, uvFreeCb); + // taosMemoryFree(req->data); taosMemoryFree(req); } @@ -349,9 +354,8 @@ static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); - tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), - ntohs(pConn->locaddr.sin_port)); + tDebug("server conn %p %s is sent to %s:%d, local info: %s:%d", pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), + ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); pHead->msgLen = htonl(len); wb->base = msg; @@ -429,11 +433,39 @@ void uvWorkerAsyncCb(uv_async_t* handle) { (*transAsyncHandle[msg->type])(msg, pThrd); } } +static void uvWalkCb(uv_handle_t* handle, void* arg) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + // uv_unref(handle); + tDebug("handle: %p -----test----", handle); + } +} +#define MAKE_VALGRIND_HAPPY(loop) \ + do { \ + uv_walk(loop, uvWalkCb, NULL); \ + uv_run(loop, UV_RUN_DEFAULT); \ + uv_loop_close(loop); \ + } while (0); + static void uvAcceptAsyncCb(uv_async_t* async) { SServerObj* srv = async->data; tDebug("close server port %d", srv->port); - uv_close((uv_handle_t*)&srv->server, NULL); - uv_stop(srv->loop); + uv_walk(srv->loop, uvWalkCb, NULL); + // uv_close((uv_handle_t*)async, NULL); + // uv_close((uv_handle_t*)&srv->server, NULL); + // uv_stop(srv->loop); + // uv_print_all_handles(srv->loop, stderr); + // int ref = uv_loop_alive(srv->loop); + // assert(ref == 0); + // tError("active size %d", ref); + // uv_stop(srv->loop); + // uv_run(srv->loop, UV_RUN_DEFAULT); + // fprintf(stderr, "------------------------------------"); + // uv_print_all_handles(srv->loop, stderr); + + // int ret = uv_loop_close(srv->loop); + // tError("(loop)->active_reqs.count: %d, ret: %d", (srv->loop)->active_reqs.count, ret); + // assert(ret == 0); } static void uvShutDownCb(uv_shutdown_t* req, int status) { @@ -455,16 +487,16 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) { if (uv_accept(stream, (uv_stream_t*)cli) == 0) { uv_write_t* wr = (uv_write_t*)taosMemoryMalloc(sizeof(uv_write_t)); - + wr->data = cli; uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThreads; tTrace("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); } else { uv_close((uv_handle_t*)cli, NULL); - taosMemoryFree(cli); } } void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { @@ -474,7 +506,10 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tError("read error %s", uv_err_name(nread)); } // TODO(log other failure reason) - // uv_close((uv_handle_t*)q, NULL); + tError("failed to create connect: %p", q); + taosMemoryFree(buf->base); + uv_close((uv_handle_t*)q, NULL); + // taosMemoryFree(q); return; } // free memory allocated by @@ -650,6 +685,7 @@ static void uvDestroyConn(uv_handle_t* handle) { if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { tTrace("work thread quit"); + // uv_walk(thrd->loop, uvWalkCb, NULL); uv_loop_close(thrd->loop); uv_stop(thrd->loop); } @@ -713,6 +749,7 @@ End: void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { + // uv_walk(thrd->loop, uvWalkCb, NULL); uv_loop_close(thrd->loop); uv_stop(thrd->loop); } else { @@ -765,8 +802,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { return; } taosThreadJoin(pThrd->thread, NULL); - taosMemoryFree(pThrd->loop); + // MAKE_VALGRIND_HAPPY(pThrd->loop); transDestroyAsyncPool(pThrd->asyncPool); + taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { @@ -784,6 +822,8 @@ void transCloseServer(void* arg) { uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); + MAKE_VALGRIND_HAPPY(srv->loop); + for (int i = 0; i < srv->numOfThreads; i++) { sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); From dcf06061cd10b130733e32491a4deea07d3ab279 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 18 Apr 2022 23:19:26 +0800 Subject: [PATCH 12/31] fix mem leak --- source/dnode/mgmt/implement/src/dmHandle.c | 8 ++++---- source/libs/transport/src/transCli.c | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 93b24535a9..1179db8ae2 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -36,12 +36,12 @@ static int32_t dmProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { } } else { SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen > 0 && - tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { + if (pRsp->pCont != NULL && pRsp->contLen > 0 && tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { pDnode->data.dnodeVer = statusRsp.dnodeVer; dmUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); dmUpdateEps(pDnode, statusRsp.pDnodeEps); } + rpcFreeCont(pRsp->pCont); tFreeSStatusRsp(&statusRsp); } @@ -76,7 +76,7 @@ void dmSendStatusReq(SDnode *pDnode) { req.pVloads = info.pVloads; int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); - void *pHead = rpcMallocCont(contLen); + void * pHead = rpcMallocCont(contLen); tSerializeSStatusReq(pHead, contLen, &req); tFreeSStatusReq(&req); @@ -101,7 +101,7 @@ int32_t dmProcessGrantRsp(SDnode *pDnode, SNodeMsg *pMsg) { } int32_t dmProcessConfigReq(SDnode *pDnode, SNodeMsg *pMsg) { - SRpcMsg *pReq = &pMsg->rpcMsg; + SRpcMsg * pReq = &pMsg->rpcMsg; SDCfgDnodeReq *pCfg = pReq->pCont; dError("config req is received, but not supported yet"); return TSDB_CODE_OPS_NOT_SUPPORT; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9dc12a3dec..1663c0637c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -471,7 +471,8 @@ static void addConnToPool(void* pool, SCliConn* conn) { assert(!QUEUE_IS_EMPTY(&plist->conn)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - SCliConn* conn = handle->data; + SCliConn* conn = handle->data; + tError("oh no %p", conn); SConnBuffer* pBuf = &conn->readBuf; transAllocBuffer(pBuf, buf); } From ed410118634c44124cccbecd370675644a2ce726 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 18 Apr 2022 23:20:45 +0800 Subject: [PATCH 13/31] fix mem leak --- source/libs/transport/src/transCli.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1663c0637c..9dc12a3dec 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -471,8 +471,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { assert(!QUEUE_IS_EMPTY(&plist->conn)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - SCliConn* conn = handle->data; - tError("oh no %p", conn); + SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; transAllocBuffer(pBuf, buf); } From 261037f34118605d7333166918605129ea989a53 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 01:57:46 +0000 Subject: [PATCH 14/31] add dbname on vnode cfg --- source/dnode/vnode/inc/vnode.h | 3 + source/dnode/vnode/src/inc/vnd.h | 2 + source/dnode/vnode/src/vnd/vnodeCfg.c | 102 ++++++++++++++++++++++- source/dnode/vnode/src/vnd/vnodeCommit.c | 74 ---------------- 4 files changed, 105 insertions(+), 76 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 86e876ecba..12e22154e8 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -40,6 +40,8 @@ typedef struct SVnode SVnode; typedef struct STsdbCfg STsdbCfg; // todo: remove typedef struct SVnodeCfg SVnodeCfg; +extern const SVnodeCfg vnodeCfgDefault; + int vnodeInit(int nthreads); void vnodeCleanup(); int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); @@ -133,6 +135,7 @@ struct STsdbCfg { struct SVnodeCfg { int32_t vgId; + char dbname[TSDB_DB_NAME_LEN]; uint64_t dbId; uint64_t wsize; uint64_t ssize; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 6be6c07e30..fa3cf65e60 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -31,6 +31,8 @@ extern "C" { // clang-format on // vnodeCfg ==================== +int vnodeEncodeConfig(const void* pObj, SJson* pJson); +int vnodeDecodeConfig(const SJson* pJson, void* pObj); // vnodeModule ==================== int vnodeScheduleTask(int (*execute)(void*), void* arg); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index a5596dc998..e4508e6442 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -15,14 +15,112 @@ #include "vnodeInt.h" -const SVnodeCfg vnodeCfgDefault = { - .wsize = 96 * 1024 * 1024, .ssize = 1 * 1024 * 1024, .lsize = 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; +const SVnodeCfg vnodeCfgDefault = {.vgId = -1, + .dbname = "", + .dbId = 0, + .wsize = 96 * 1024 * 1024, + .ssize = 1 * 1024 * 1024, + .lsize = 1024, + .isHeapAllocator = false, + .ttl = 0, + .keep = 0, + .streamMode = 0, + .isWeak = 0, + .tsdbCfg = {.precision = TWO_STAGE_COMP, + .update = 0, + .compression = 2, + .days = 10, + .minRows = 100, + .maxRows = 4096, + .keep2 = 3650, + .keep0 = 3650, + .keep1 = 3650}, + .walCfg = {.level = TAOS_WAL_WRITE}, + .hashBegin = 0, + .hashEnd = 0, + .hashMethod = 0}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { // TODO return 0; } +int vnodeEncodeConfig(const void *pObj, SJson *pJson) { + const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; + + if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1; + if (tjsonAddStringToObject(pJson, "dbname", pCfg->dbname) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; + + return 0; +} + +int vnodeDecodeConfig(const SJson *pJson, void *pObj) { + SVnodeCfg *pCfg = (SVnodeCfg *)pObj; + + if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1; + if (tjsonGetStringValue(pJson, "dbname", pCfg->dbname) < 0) return -1; + if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1; + if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1; + if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1; + if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1; + if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1; + if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1; + if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1; + if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1; + if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; + if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; + if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; + if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; + if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; + if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; + if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; + if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; + if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; + if (tjsonGetNumberValue(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; + if (tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; + if (tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; + if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; + if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; + + return 0; +} + int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) { uint32_t hashValue = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3ece3c054e..55e4c5110a 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -181,80 +181,6 @@ static int vnodeEndCommit(SVnode *pVnode) { static FORCE_INLINE void vnodeWaitCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); } -static int vnodeEncodeConfig(const void *pObj, SJson *pJson) { - const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; - - if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; - - return 0; -} - -static int vnodeDecodeConfig(const SJson *pJson, void *pObj) { - SVnodeCfg *pCfg = (SVnodeCfg *)pObj; - - if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1; - if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1; - if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1; - if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1; - if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1; - if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1; - if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1; - if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1; - if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1; - if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1; - if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; - if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; - if (tjsonGetNumberValue(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; - if (tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; - if (tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; - if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; - if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; - - return 0; -} - static int vnodeEncodeState(const void *pObj, SJson *pJson) { const SVState *pState = (SVState *)pObj; From 763a5550d0ffa4acb8858a92687bf87aa63285cb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 02:04:20 +0000 Subject: [PATCH 15/31] pass db name in --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 12 ++--- source/dnode/vnode/src/vnd/vnodeCfg.c | 50 +++++++++++---------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 7eb08a8f4f..751edd6f98 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -82,13 +82,13 @@ int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) { } static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { + memcpy(pCfg, &vnodeCfgDefault, sizeof(SVnodeCfg)); + pCfg->vgId = pCreate->vgId; + strcpy(pCfg->dbname, pCreate->db); pCfg->wsize = pCreate->cacheBlockSize * 1024 * 1024; pCfg->ssize = 1024; pCfg->lsize = 1024 * 1024; - pCfg->isHeapAllocator = true; - pCfg->ttl = 4; - pCfg->keep = pCreate->daysToKeep0; pCfg->streamMode = pCreate->streamMode; pCfg->isWeak = true; pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; @@ -96,12 +96,6 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbCfg.keep1 = pCreate->daysToKeep0; pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; pCfg->tsdbCfg.retentions = pCreate->pRetensions; - pCfg->walCfg.level = TAOS_WAL_WRITE; - pCfg->walCfg.fsyncPeriod = 0; - pCfg->walCfg.retentionPeriod = 0; - pCfg->walCfg.retentionSize = 0; - pCfg->walCfg.rollPeriod = 0; - pCfg->walCfg.segSize = 0; pCfg->walCfg.vgId = pCreate->vgId; pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index e4508e6442..625a2b3aed 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -15,30 +15,32 @@ #include "vnodeInt.h" -const SVnodeCfg vnodeCfgDefault = {.vgId = -1, - .dbname = "", - .dbId = 0, - .wsize = 96 * 1024 * 1024, - .ssize = 1 * 1024 * 1024, - .lsize = 1024, - .isHeapAllocator = false, - .ttl = 0, - .keep = 0, - .streamMode = 0, - .isWeak = 0, - .tsdbCfg = {.precision = TWO_STAGE_COMP, - .update = 0, - .compression = 2, - .days = 10, - .minRows = 100, - .maxRows = 4096, - .keep2 = 3650, - .keep0 = 3650, - .keep1 = 3650}, - .walCfg = {.level = TAOS_WAL_WRITE}, - .hashBegin = 0, - .hashEnd = 0, - .hashMethod = 0}; +const SVnodeCfg vnodeCfgDefault = { + .vgId = -1, + .dbname = "", + .dbId = 0, + .wsize = 96 * 1024 * 1024, + .ssize = 1 * 1024 * 1024, + .lsize = 1024, + .isHeapAllocator = false, + .ttl = 0, + .keep = 0, + .streamMode = 0, + .isWeak = 0, + .tsdbCfg = {.precision = TWO_STAGE_COMP, + .update = 0, + .compression = 2, + .days = 10, + .minRows = 100, + .maxRows = 4096, + .keep2 = 3650, + .keep0 = 3650, + .keep1 = 3650}, + .walCfg = + {.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE}, + .hashBegin = 0, + .hashEnd = 0, + .hashMethod = 0}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { // TODO From 2305b05869c8238998167798ee0da6a275dab1b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Apr 2022 10:12:30 +0800 Subject: [PATCH 16/31] refactor(query): do some internal refactor for executor. --- source/libs/executor/inc/executorimpl.h | 116 ++++++++++++------------ source/libs/executor/src/executorimpl.c | 73 +++++++++------ source/libs/function/src/builtinsimpl.c | 81 ++++++++++++----- 3 files changed, 163 insertions(+), 107 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index fa86cd2a18..230ccf6ace 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -404,25 +404,29 @@ typedef struct SAggSupporter { SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; -typedef struct STableIntervalOperatorInfo { - SOptrBasicInfo binfo; // basic info - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - STimeWindow win; // query time range - bool timeWindowInterpo; // interpolation needed or not - char** pRow; // previous row/tuple of already processed datablock - SAggSupporter aggSup; // aggregate supporter - STableQueryInfo* pCurrent; // current tableQueryInfo struct - int32_t order; // current SSDataBlock scan order - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] - SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. +typedef struct STimeWindowSupp { + int8_t calTrigger; + int64_t waterMark; SColumnInfoData timeWindowData; // query time window info for scalar function execution. - double watermark; // water mark - int32_t trigger; // trigger model +} STimeWindowAggSupp; + +typedef struct STableIntervalOperatorInfo { + SOptrBasicInfo binfo; // basic info + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + STimeWindow win; // query time range + bool timeWindowInterpo; // interpolation needed or not + char** pRow; // previous row/tuple of already processed datablock + SAggSupporter aggSup; // aggregate supporter + STableQueryInfo* pCurrent; // current tableQueryInfo struct + int32_t order; // current SSDataBlock scan order + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. + STimeWindowAggSupp twAggSup; } STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -440,19 +444,19 @@ typedef struct SAggOperatorInfo { } SAggOperatorInfo; typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SSDataBlock* existDataBlock; - SArray* pPseudoColInfo; - SLimit limit; - SLimit slimit; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SSDataBlock* existDataBlock; + SArray* pPseudoColInfo; + SLimit limit; + SLimit slimit; - uint64_t groupId; - int64_t curSOffset; - int64_t curGroupOutput; + uint64_t groupId; + int64_t curSOffset; + int64_t curGroupOutput; - int64_t curOffset; - int64_t curOutput; + int64_t curOffset; + int64_t curOutput; } SProjectOperatorInfo; typedef struct SFillOperatorInfo { @@ -467,10 +471,10 @@ typedef struct SFillOperatorInfo { } SFillOperatorInfo; typedef struct { - char* pData; - bool isNull; - int16_t type; - int32_t bytes; + char* pData; + bool isNull; + int16_t type; + int32_t bytes; } SGroupKeys, SStateKeys; typedef struct SGroupbyOperatorInfo { @@ -489,9 +493,9 @@ typedef struct SGroupbyOperatorInfo { } SGroupbyOperatorInfo; typedef struct SDataGroupInfo { - uint64_t groupId; - int64_t numOfRows; - SArray* pPageList; + uint64_t groupId; + int64_t numOfRows; + SArray* pPageList; } SDataGroupInfo; // The sort in partition may be needed later. @@ -506,9 +510,8 @@ typedef struct SPartitionOperatorInfo { SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file int32_t rowCapacity; // maximum number of rows for each buffer page int32_t* columnOffset; // start position for each column data - - void* pGroupIter; // group iterator - int32_t pageIndex; // page index of current group + void* pGroupIter; // group iterator + int32_t pageIndex; // page index of current group } SPartitionOperatorInfo; typedef struct SWindowRowsSup { @@ -519,13 +522,13 @@ typedef struct SWindowRowsSup { } SWindowRowsSup; typedef struct SSessionAggOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SGroupResInfo groupResInfo; - SWindowRowsSup winSup; - bool reptScan; // next round scan - int64_t gap; // session window gap - SColumnInfoData timeWindowData; // query time window info for scalar function execution. + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SGroupResInfo groupResInfo; + SWindowRowsSup winSup; + bool reptScan; // next round scan + int64_t gap; // session window gap + STimeWindowAggSupp twAggSup; } SSessionAggOperatorInfo; typedef struct STimeSliceOperatorInfo { @@ -535,14 +538,14 @@ typedef struct STimeSliceOperatorInfo { } STimeSliceOperatorInfo; typedef struct SStateWindowOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SGroupResInfo groupResInfo; - SWindowRowsSup winSup; - int32_t colIndex; // start row index - bool hasKey; - SStateKeys stateKey; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SGroupResInfo groupResInfo; + SWindowRowsSup winSup; + int32_t colIndex; // start row index + bool hasKey; + SStateKeys stateKey; + STimeWindowAggSupp twAggSup; // bool reptScan; } SStateWindowOperatorInfo; @@ -640,10 +643,11 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlot, - const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, @@ -656,7 +660,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp SInterval* pInterval, SSDataBlock* pResBlock, int32_t fillType, char* fillVal, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SExecTaskInfo* pTaskInfo, diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0fdb523e64..eaf09237de 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1097,6 +1097,8 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->totalRows = pBlock->info.rows; pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; + + pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { if (createDummyCol) { @@ -1516,8 +1518,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // window start key interpolation doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false); - updateTimeWindowInfo(&pInfo->timeWindowData, &win, true); - doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; while (1) { @@ -1548,8 +1550,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false); - updateTimeWindowInfo(&pInfo->timeWindowData, &nextWin, true); - doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } if (pInfo->timeWindowInterpo) { @@ -1620,8 +1622,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator } // pInfo->numOfRows data belong to the current session window - updateTimeWindowInfo(&pInfo->timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j); @@ -1637,8 +1639,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - updateTimeWindowInfo(&pInfo->timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { @@ -5338,8 +5340,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - updateTimeWindowInfo(&pInfo->timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j); @@ -5355,8 +5357,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - updateTimeWindowInfo(&pInfo->timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { @@ -5863,30 +5865,30 @@ _error: } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlot, - const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = pTaskInfo->execModel; - - pInfo->win = pTaskInfo->window; - pInfo->win.skey = 0; - pInfo->win.ekey = INT64_MAX; - - pInfo->primaryTsIndex = primaryTsSlot; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->win.skey = 0; + pInfo->win.ekey = INT64_MAX; + pInfo->primaryTsIndex = primaryTsSlotId; + pInfo->twAggSup = *pTwAggSupp; int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); - initExecTimeWindowInfo(&pInfo->timeWindowData, &pInfo->win); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { @@ -5955,7 +5957,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* return NULL; } -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSup, + SExecTaskInfo* pTaskInfo) { SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5969,6 +5972,9 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExpr, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + pInfo->twAggSup = *pTwAggSup; + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; pOperator->blockingOptr = true; @@ -5992,7 +5998,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo) { SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -6008,8 +6014,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo goto _error; } + pInfo->twAggSup = *pTwAggSupp; initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - initExecTimeWindowInfo(&pInfo->timeWindowData, &pTaskInfo->window); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->gap = gap; pInfo->binfo.pRes = pResBlock; @@ -6559,8 +6566,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision }; + STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType}; + int32_t primaryTsSlotId = ((SColumnNode*) pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, pTableGroupInfo, pTaskInfo); + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, pTableGroupInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { SSortPhysiNode* pSortPhyNode = (SSortPhysiNode*)pPhyNode; @@ -6571,9 +6580,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; + STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, pTaskInfo); + pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*) pPhyNode; SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); @@ -6584,9 +6595,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW == type) { SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*) pPhyNode; + STimeWindowAggSupp as = {.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType}; + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, pTaskInfo); + pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*) pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 0d7e984ae7..3b674759a3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -771,13 +771,9 @@ bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { return true; } -// TODO fix this -// This ordinary first function only handle the data block in ascending order +// This ordinary first function does not care if current scan is ascending order or descending order scan +// the OPTIMIZED version of first function will only handle the ascending order scan int32_t firstFunction(SqlFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return 0; - } - int32_t numOfElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -786,29 +782,72 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; + int32_t bytes = pInputCol->info.bytes; + // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); return 0; } - // Check for the first not null data - for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, NULL)) { - continue; + SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet)? pInput->pColumnDataAgg[0]:NULL; + + TSKEY startKey = *(int64_t*)(pInput->pPTS ? colDataGetData(pInput->pPTS, 0) : 0); + TSKEY endKey = *(int64_t*)(pInput->pPTS ? colDataGetData(pInput->pPTS, pInput->totalRows - 1) : 0); + + int32_t blockDataOrder = (startKey <= endKey)? TSDB_ORDER_ASC:TSDB_ORDER_DESC; + + if (blockDataOrder == TSDB_ORDER_ASC) { + // filter according to current result firstly + if (pResInfo->numOfRes > 0) { + TSKEY ts = *(TSKEY*)(buf + bytes); + if (ts < startKey) { + return TSDB_CODE_SUCCESS; + } } - char* data = colDataGetData(pInputCol, i); - memcpy(buf, data, pInputCol->info.bytes); - // TODO handle the subsidary value -// if (pCtx->ptsList != NULL) { -// TSKEY k = GET_TS_DATA(pCtx, i); -// DO_UPDATE_TAG_COLUMNS(pCtx, k); -// } + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } - pResInfo->complete = true; - numOfElems++; - break; + char* data = colDataGetData(pInputCol, i); + + TSKEY cts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { + memcpy(buf, data, bytes); + *(TSKEY*)(buf + bytes) = cts; +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + + numOfElems++; + } + } else { + // in case of descending order time stamp serial, which usually happens as the results of the nest query, + // all data needs to be check. + if (pResInfo->numOfRes > 0) { + TSKEY ts = *(TSKEY*)(buf + bytes); + if (ts < endKey) { + return TSDB_CODE_SUCCESS; + } + } + + for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + char* data = colDataGetData(pInputCol, i); + + TSKEY cts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { + memcpy(buf, data, bytes); + *(TSKEY*)(buf + bytes) = cts; +// DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + + numOfElems++; + } } SET_VAL(pResInfo, numOfElems, 1); @@ -829,7 +868,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) { SColumnInfoData* pInputCol = pInput->pData[0]; // All null data column, return directly. - if (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows) { + if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); return 0; } From d1cb2ffdf83ba6240bf8c92a7b24f1eb02ab86b2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 19 Apr 2022 10:12:33 +0800 Subject: [PATCH 17/31] refactor(cluster): remove the online field while execute 'show vgroups' --- source/dnode/mnode/impl/src/mndInfoSchema.c | 48 ++++++++++----------- source/dnode/mnode/impl/src/mndVgroup.c | 11 ----- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index f626332fc2..d84c5f84cf 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -41,8 +41,8 @@ static const SInfosTableSchema mnodesSchema[] = { static const SInfosTableSchema modulesSchema[] = { {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "endpoint", .bytes = 134 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "module", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SInfosTableSchema qnodesSchema[] = { @@ -145,9 +145,9 @@ static const SInfosTableSchema userTblsSchema[] = { }; static const SInfosTableSchema userTblDistSchema[] = { - {.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "distributed_histogram", .bytes = 500 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -168,35 +168,33 @@ static const SInfosTableSchema userUsersSchema[] = { }; static const SInfosTableSchema grantsSchema[] = { - {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SInfosTableSchema vgroupsSchema[] = { {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; @@ -206,7 +204,7 @@ static const SInfosTableSchema topicSchema[] = { {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "sql", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, }; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index dcb48c95a7..75fe409d2a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -537,17 +537,6 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->numOfTables, false); - // status - char buf[10] = {0}; - STR_TO_VARSTR(buf, "ready"); // TODO - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, buf, false); - - // onlines - int32_t onlines = pVgroup->replica; - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&onlines, false); - // default 3 replica for (int32_t i = 0; i < 3; ++i) { From 3a50257aa3fbdae5a4fb72ea243319e216387336 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 02:18:11 +0000 Subject: [PATCH 18/31] refact --- include/common/tmsg.h | 1 - source/common/src/tmsg.c | 2 - source/dnode/mnode/impl/src/mndStb.c | 1 - source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +- source/libs/parser/src/parInsert.c | 269 +++++++++++++------------ source/libs/parser/src/parTranslater.c | 4 - 6 files changed, 138 insertions(+), 144 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8c30f7d280..67369bb42c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1432,7 +1432,6 @@ typedef struct { typedef struct SVCreateTbReq { int64_t ver; // use a general definition - char* dbFName; char* name; uint32_t ttl; uint32_t keep; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6df2341af4..5c97743a62 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -398,7 +398,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->ver); - tlen += taosEncodeString(buf, pReq->dbFName); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -467,7 +466,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI64(buf, &(pReq->ver)); - buf = taosDecodeString(buf, &(pReq->dbFName)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2aa20df2e6..fa24501651 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -349,7 +349,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt SVCreateTbReq req = {0}; req.ver = 0; - req.dbFName = dbFName; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 288241eb66..928a2d4e4b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -217,7 +217,6 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) { taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds); taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); } - taosMemoryFree(vCreateTbReq.dbFName); taosMemoryFree(vCreateTbReq.name); return 0; @@ -233,7 +232,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR char tableFName[TSDB_TABLE_FNAME_LEN]; SMsgHead *pHead = (SMsgHead *)pMsg->pCont; - sprintf(tableFName, "%s.%s", pCreateTbReq->dbFName, pCreateTbReq->name); + sprintf(tableFName, "%s.%s", pVnode->config.dbname, pCreateTbReq->name); int32_t code = vnodeValidateTableHash(&pVnode->config, tableFName); if (code) { @@ -249,7 +248,6 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR } // TODO: to encapsule a free API taosMemoryFree(pCreateTbReq->name); - taosMemoryFree(pCreateTbReq->dbFName); if (pCreateTbReq->type == TD_SUPER_TABLE) { taosMemoryFree(pCreateTbReq->stbCfg.pSchema); taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); @@ -298,7 +296,6 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) { taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam->pFuncIds); taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); } - taosMemoryFree(vAlterTbReq.dbFName); taosMemoryFree(vAlterTbReq.name); return 0; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index acc597d61b..85a42387c4 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -15,57 +15,57 @@ #include "parInsertData.h" #include "parInt.h" -#include "parUtil.h" #include "parToken.h" +#include "parUtil.h" #include "tglobal.h" #include "ttime.h" #include "ttypes.h" -#define NEXT_TOKEN(pSql, sToken) \ - do { \ - int32_t index = 0; \ +#define NEXT_TOKEN(pSql, sToken) \ + do { \ + int32_t index = 0; \ sToken = tStrGetToken(pSql, &index, false); \ - pSql += index; \ + pSql += index; \ } while (0) -#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \ - do { \ - int32_t index = 0; \ +#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \ + do { \ + int32_t index = 0; \ sToken = tStrGetToken(pSql, &index, true); \ - pSql += index; \ + pSql += index; \ } while (0) #define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ - do { \ - sToken = tStrGetToken(pSql, &index, false); \ + do { \ + sToken = tStrGetToken(pSql, &index, false); \ } while (0) -#define CHECK_CODE(expr) \ - do { \ - int32_t code = expr; \ +#define CHECK_CODE(expr) \ + do { \ + int32_t code = expr; \ if (TSDB_CODE_SUCCESS != code) { \ - return code; \ - } \ + return code; \ + } \ } while (0) typedef struct SInsertParseContext { - SParseContext* pComCxt; // input - char *pSql; // input - SMsgBuf msg; // input - STableMeta* pTableMeta; // each table - SParsedDataColInfo tags; // each table - SKVRowBuilder tagsBuilder; // each table - SVCreateTbReq createTblReq; // each table - SHashObj* pVgroupsHashObj; // global - SHashObj* pTableBlockHashObj; // global - SHashObj* pSubTableHashObj; // global - SArray* pTableDataBlocks; // global - SArray* pVgDataBlocks; // global - int32_t totalNum; + SParseContext* pComCxt; // input + char* pSql; // input + SMsgBuf msg; // input + STableMeta* pTableMeta; // each table + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SVCreateTbReq createTblReq; // each table + SHashObj* pVgroupsHashObj; // global + SHashObj* pTableBlockHashObj; // global + SHashObj* pSubTableHashObj; // global + SArray* pTableDataBlocks; // global + SArray* pVgDataBlocks; // global + int32_t totalNum; SVnodeModifOpStmt* pOutput; } SInsertParseContext; -typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param); +typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; @@ -151,7 +151,7 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD } char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); - if (NULL != p) { // db.table + if (NULL != p) { // db.table int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->acctId); strncpy(fullDbName + n, pStname->z, p - pStname->z); strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); @@ -168,14 +168,14 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar const char* msg2 = "invalid database name"; const char* msg3 = "db is not specified"; - int32_t code = TSDB_CODE_SUCCESS; - char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, true); + int32_t code = TSDB_CODE_SUCCESS; + char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, true); - if (p != NULL) { // db has been specified in sql string so we ignore current db path + if (p != NULL) { // db has been specified in sql string so we ignore current db path assert(*p == TS_PATH_DELIMITER[0]); int32_t dbLen = p - pTableName->z; - char name[TSDB_DB_FNAME_LEN] = {0}; + char name[TSDB_DB_FNAME_LEN] = {0}; strncpy(name, pTableName->z, dbLen); dbLen = strdequote(name); @@ -185,9 +185,9 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar } int32_t tbLen = pTableName->n - dbLen - 1; - char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(tbname, p + 1, tbLen); - /*tbLen = */strdequote(tbname); + /*tbLen = */ strdequote(tbname); code = tNameFromString(pName, tbname, T_NAME_TABLE); if (code != 0) { @@ -225,28 +225,26 @@ static int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pPar static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool isStb) { SParseContext* pBasicCtx = pCxt->pComCxt; - SName name = {0}; - createSName(&name, pTname, pBasicCtx, &pCxt->msg); + SName name = {0}; + createSName(&name, pTname, pBasicCtx, &pCxt->msg); if (isStb) { - CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); + CHECK_CODE(catalogGetSTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, + &pCxt->pTableMeta)); } else { - CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); + CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, + &pCxt->pTableMeta)); } SVgroupInfo vg; - CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - + return TSDB_CODE_SUCCESS; } -static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { - return getTableMetaImpl(pCxt, pTname, false); -} - -static int32_t getSTableMeta(SInsertParseContext* pCxt, SToken* pTname) { - return getTableMetaImpl(pCxt, pTname, true); -} +static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { return getTableMetaImpl(pCxt, pTname, false); } +static int32_t getSTableMeta(SInsertParseContext* pCxt, SToken* pTname) { return getTableMetaImpl(pCxt, pTname, true); } static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { @@ -259,24 +257,24 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS } static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { - SSubmitReq* submit = (SSubmitReq*)blocks->pData; - submit->header.vgId = htonl(blocks->vg.vgId); - submit->header.contLen = htonl(blocks->size); - submit->length = submit->header.contLen; - submit->numOfBlocks = htonl(blocks->numOfTables); - SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); - int32_t numOfBlocks = blocks->numOfTables; - while (numOfBlocks--) { - int32_t dataLen = blk->dataLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->padding = htonl(blk->padding); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); - } + SSubmitReq* submit = (SSubmitReq*)blocks->pData; + submit->header.vgId = htonl(blocks->vg.vgId); + submit->header.contLen = htonl(blocks->size); + submit->length = submit->header.contLen; + submit->numOfBlocks = htonl(blocks->numOfTables); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + int32_t numOfBlocks = blocks->numOfTables; + while (numOfBlocks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->suid = htobe64(blk->suid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } } static int32_t buildOutput(SInsertParseContext* pCxt) { @@ -287,7 +285,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -301,13 +299,13 @@ static int32_t buildOutput(SInsertParseContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { +static int32_t checkTimestamp(STableDataBlocks* pDataBlocks, const char* start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { return TSDB_CODE_SUCCESS; } - TSKEY k = *(TSKEY *)start; + TSKEY k = *(TSKEY*)start; if (k <= pDataBlocks->prevTS) { pDataBlocks->ordered = false; } @@ -316,12 +314,12 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) return TSDB_CODE_SUCCESS; } -static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time, SMsgBuf* pMsgBuf) { - int32_t index = 0; - SToken sToken; - int64_t interval; - int64_t ts = 0; - char* pTokenEnd = *end; +static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { + int32_t index = 0; + SToken sToken; + int64_t interval; + int64_t ts = 0; + char* pTokenEnd = *end; if (pToken->type == TK_NOW) { ts = taosGetTimestamp(timePrec); @@ -330,7 +328,7 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time } else if (pToken->type == TK_NK_INTEGER) { bool isSigned = false; toInteger(pToken->z, pToken->n, 10, &ts, &isSigned); - } else { // parse the RFC-3339/ISO-8601 timestamp format string + } else { // parse the RFC-3339/ISO-8601 timestamp format string if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } @@ -340,7 +338,7 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time for (int k = pToken->n; pToken->z[k] != '\0'; k++) { if (pToken->z[k] == ' ' || pToken->z[k] == '\t') continue; - if (pToken->z[k] == '(' && pToken->z[k + 1] == ')') { //for insert NOW()/TODAY() + if (pToken->z[k] == '(' && pToken->z[k + 1] == ')') { // for insert NOW()/TODAY() *end = pTokenEnd = &pToken->z[k + 2]; k++; continue; @@ -391,8 +389,10 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time } static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { - if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && - pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && pToken->type != TK_NK_BIN) || + if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && + pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && + pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && + pToken->type != TK_NK_BIN) || (pToken->n == 0) || (pToken->type == TK_NK_RP)) { return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } @@ -408,7 +408,7 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha } // delete escape character: \\, \', \" - char delim = pToken->z[0]; + char delim = pToken->z[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < pToken->n - 1; ++k) { @@ -431,12 +431,12 @@ static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, cha return TSDB_CODE_SUCCESS; } -static bool isNullStr(SToken *pToken) { +static bool isNullStr(SToken* pToken) { return (pToken->type == TK_NULL) || ((pToken->type == TK_NK_STRING) && (pToken->n != 0) && (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); } -static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPtr) { +static FORCE_INLINE int32_t toDouble(SToken* pToken, double* value, char** endPtr) { errno = 0; *value = strtold(pToken->z, endPtr); @@ -448,9 +448,10 @@ static FORCE_INLINE int32_t toDouble(SToken *pToken, double *value, char **endPt return pToken->type; } -static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { +static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, + _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf) { int64_t iv; - char *endptr = NULL; + char* endptr = NULL; bool isSigned = false; int32_t code = checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf); @@ -497,7 +498,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return func(pMsgBuf, &tmpVal, pSchema->bytes, param); } - case TSDB_DATA_TYPE_UTINYINT:{ + case TSDB_DATA_TYPE_UTINYINT: { if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv, &isSigned)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); } else if (!IS_VALID_UTINYINT(iv)) { @@ -571,7 +572,8 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } - if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) { + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); } float tmpVal = (float)dv; @@ -657,15 +659,15 @@ static FORCE_INLINE int32_t MemRowAppend(SMsgBuf* pMsgBuf, const void* value, in static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; - pColList->numOfBound = 0; + pColList->numOfBound = 0; pColList->boundNullLen = 0; memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); for (col_id_t i = 0; i < nCols; ++i) { pColList->cols[i].valStat = VAL_STAT_NONE; } - SToken sToken; - bool isOrdered = true; + SToken sToken; + bool isOrdered = true; col_id_t lastColIdx = -1; // last column found while (1) { NEXT_TOKEN(pCxt->pSql, sToken); @@ -729,13 +731,13 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } typedef struct SKvParam { - SKVRowBuilder *builder; - SSchema *schema; + SKVRowBuilder* builder; + SSchema* schema; char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; -static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, void *param) { - SKvParam* pa = (SKvParam*) param; +static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { + SKvParam* pa = (SKvParam*)param; int8_t type = pa->schema->type; int16_t colId = pa->schema->colId; @@ -749,7 +751,8 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void *value, int32_t len, voi if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { char buf[512] = {0}; snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value);; + return buildSyntaxErrMsg(pMsgBuf, buf, value); + ; } varDataSetLen(pa->buf, output); @@ -765,7 +768,6 @@ static int32_t buildCreateTbReq(SInsertParseContext* pCxt, const SName* pName, S char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pName, dbFName); pCxt->createTblReq.type = TD_CHILD_TABLE; - pCxt->createTblReq.dbFName = strdup(dbFName); pCxt->createTblReq.name = strdup(pName->tname); pCxt->createTblReq.ctbCfg.suid = pCxt->pTableMeta->suid; pCxt->createTblReq.ctbCfg.pTag = row; @@ -780,13 +782,14 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } SKvParam param = {.builder = &pCxt->tagsBuilder}; - SToken sToken; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + SToken sToken; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 + SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i] - 1]; // colId starts with 1 param.schema = pTagSchema; - CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); + CHECK_CODE( + parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); } SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); @@ -822,7 +825,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) createSName(&name, pTbnameToken, pCxt->pComCxt, &pCxt->msg); char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&name, tbFName); - int32_t len = strlen(tbFName); + int32_t len = strlen(tbFName); STableMeta** pMeta = taosHashGet(pCxt->pSubTableHashObj, tbFName, len); if (NULL != pMeta) { return cloneTableMeta(*pMeta, &pCxt->pTableMeta); @@ -864,17 +867,18 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) return TSDB_CODE_SUCCESS; } -static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) { +static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, + char* tmpTokenBuf) { SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlocks->rowBuilder; STSRow* row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size); // skip the SSubmitBlk header tdSRowResetBuf(pBuilder, row); - bool isParseBindParam = false; - SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta); + bool isParseBindParam = false; + SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta); SMemParam param = {.rb = pBuilder}; - SToken sToken = {0}; + SToken sToken = {0}; // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); @@ -885,7 +889,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { TSKEY tsKey = TD_ROW_KEY(row); - if (checkTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) { + if (checkTimestamp(pDataBlocks, (const char*)&tsKey) != TSDB_CODE_SUCCESS) { buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } @@ -911,11 +915,11 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, // pSql -> (field1_value, ...) [(field1_value2, ...) ...] static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) { STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); (*numOfRows) = 0; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" SToken sToken; while (1) { int32_t index = 0; @@ -934,7 +938,7 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo int32_t len = 0; CHECK_CODE(parseOneRow(pCxt, pDataBlock, tinfo.precision, &len, tmpTokenBuf)); - pDataBlock->size += extendedRowSize; //len; + pDataBlock->size += extendedRowSize; // len; NEXT_TOKEN(pCxt->pSql, sToken); if (TK_NK_RP != sToken.type) { @@ -945,19 +949,19 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo } if (0 == (*numOfRows)) { - return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); + return buildSyntaxErrMsg(&pCxt->msg, "no any data points", NULL); } return TSDB_CODE_SUCCESS; } -static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) { +static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* dataBuf) { int32_t maxNumOfRows; CHECK_CODE(allocateMemIfNeed(dataBuf, getExtendedRowSize(dataBuf), &maxNumOfRows)); int32_t numOfRows = 0; CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); - SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); + SSubmitBlk* pBlocks = (SSubmitBlk*)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf, numOfRows)) { return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767"); } @@ -968,7 +972,6 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da } static void destroyCreateSubTbReq(SVCreateTbReq* pReq) { - taosMemoryFreeClear(pReq->dbFName); taosMemoryFreeClear(pReq->name); taosMemoryFreeClear(pReq->ctbCfg.pTag); } @@ -1023,7 +1026,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // no data in the sql string anymore. if (sToken.n == 0) { if (0 == pCxt->totalNum) { - return buildInvalidOperationMsg(&pCxt->msg, "no data in sql");; + return buildInvalidOperationMsg(&pCxt->msg, "no data in sql"); + ; } break; } @@ -1031,7 +1035,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SToken tbnameToken = sToken; NEXT_TOKEN(pCxt->pSql, sToken); - // USING cluase + // USING cluase if (TK_USING == sToken.type) { CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); NEXT_TOKEN(pCxt->pSql, sToken); @@ -1039,10 +1043,11 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { CHECK_CODE(getTableMeta(pCxt, &tbnameToken)); } - STableDataBlocks *dataBuf = NULL; + STableDataBlocks* dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL, &pCxt->createTblReq)); - + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, + &dataBuf, NULL, &pCxt->createTblReq)); + if (TK_NK_LP == sToken.type) { // pSql -> field1_name, ...) CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); @@ -1071,7 +1076,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", sToken.z); } // merge according to vgId - if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { + if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && + taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } return buildOutput(pCxt); @@ -1085,19 +1091,18 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // [...]; int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { SInsertParseContext context = { - .pComCxt = pContext, - .pSql = (char*) pContext->pSql, - .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pTableMeta = NULL, - .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), - .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), - .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), - .totalNum = 0, - .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT) - }; + .pComCxt = pContext, + .pSql = (char*)pContext->pSql, + .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, + .pTableMeta = NULL, + .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), + .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), + .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false), + .totalNum = 0, + .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT)}; - if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || - NULL == context.pSubTableHashObj || NULL == context.pOutput) { + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || + NULL == context.pOutput) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a7e7e01948..5eb9815dd7 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2747,7 +2747,6 @@ static void toSchemaEx(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSch } static void destroyCreateTbReq(SVCreateTbReq* pReq) { - taosMemoryFreeClear(pReq->dbFName); taosMemoryFreeClear(pReq->name); taosMemoryFreeClear(pReq->ntbCfg.pSchema); } @@ -2784,7 +2783,6 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SVCreateTbReq req = {0}; req.type = TD_NORMAL_TABLE; - req.dbFName = strdup(dbFName); req.name = strdup(pStmt->tableName); req.ntbCfg.nCols = LIST_LENGTH(pStmt->pCols); req.ntbCfg.pSchema = taosMemoryCalloc(req.ntbCfg.nCols, sizeof(SSchema)); @@ -2843,7 +2841,6 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); - taosMemoryFreeClear(pTableReq->dbFName); taosMemoryFreeClear(pTableReq->name); if (pTableReq->type == TSDB_NORMAL_TABLE) { @@ -2929,7 +2926,6 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; - req.dbFName = strdup(dbFName); req.name = strdup(pTableName); req.ctbCfg.suid = suid; req.ctbCfg.pTag = row; From a6177f54caa4efb7aef94a22a34c18df16287776 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Apr 2022 10:28:39 +0800 Subject: [PATCH 19/31] test: disable free in hash temporarily. --- source/util/src/thash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 4874bc16d7..6d3a259079 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -33,9 +33,9 @@ #define FREE_HASH_NODE(_fp, _n) \ do { \ - if (_fp != NULL) { \ +/* if (_fp != NULL) { \ (_fp)(_n); \ - } \ + }*/ \ taosMemoryFreeClear(_n); \ } while (0); From 7895c492bac52507cb944621fd324008f801d5bf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 02:37:45 +0000 Subject: [PATCH 20/31] refact vnod3 --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 7 ++++--- source/dnode/vnode/inc/vnode.h | 4 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 22 +++++++++------------ 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 2fb29ce944..8338d20206 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -97,6 +97,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; + int64_t version; SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); if (pArray == NULL) { @@ -115,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } } - vnodePreprocessWriteReqs(pVnode->pImpl, pArray); + vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version); numOfMsgs = taosArrayGetSize(pArray); for (int32_t i = 0; i < numOfMsgs; i++) { @@ -123,7 +124,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRsp = NULL; - int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, &pRsp); + int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &pRsp); if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; tmsgSendRsp(pRsp); @@ -153,7 +154,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // todo SRpcMsg *pRsp = NULL; - (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); + // (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); } } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 12e22154e8..2a9f540ffb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -48,8 +48,8 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodeClose(SVnode *pVnode); -void vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs); -int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version); +int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg **pRsp); int vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 928a2d4e4b..e7a3e1b29c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -20,30 +20,28 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq); static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp); -void vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs) { +int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { SNodeMsg *pMsg; SRpcMsg *pRpc; + *version = pVnode->state.processed; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SNodeMsg **)taosArrayGet(pMsgs, i); pRpc = &pMsg->rpcMsg; // set request version - void *pBuf = POINTER_SHIFT(pRpc->pCont, sizeof(SMsgHead)); - int64_t ver = pVnode->state.processed++; - taosEncodeFixedI64(&pBuf, ver); - - if (walWrite(pVnode->pWal, ver, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { - // TODO: handle error - /*ASSERT(false);*/ + if (walWrite(pVnode->pWal, pVnode->state.processed++, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) { vError("vnode:%d write wal error since %s", TD_VID(pVnode), terrstr()); + return -1; } } walFsync(pVnode->pWal, false); + + return 0; } -int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg **pRsp) { void *ptr = NULL; int ret; @@ -58,9 +56,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // todo: change the interface here - int64_t ver; - taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); - if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, ver) < 0) { + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { // TODO: handle error } @@ -128,7 +124,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { break; } - pVnode->state.applied = ver; + pVnode->state.applied = version; // Check if it needs to commit if (vnodeShouldCommit(pVnode)) { From f8d1f5a813fa80ecc9d8e9052ab7630e06c084cb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 03:17:23 +0000 Subject: [PATCH 21/31] refact vnode --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 15 +++++++++--- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 26 +++++++-------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 8338d20206..777f9eb36e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -122,17 +122,26 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO for (int32_t i = 0; i < numOfMsgs; i++) { SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); SRpcMsg *pRpc = &pMsg->rpcMsg; - SRpcMsg *pRsp = NULL; + SRpcMsg rsp; - int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &pRsp); + rsp.pCont = NULL; + rsp.contLen = 0; + rsp.code = 0; + rsp.handle = pRpc->handle; + rsp.ahandle = pRpc->ahandle; + + int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp); + tmsgSendRsp(&rsp); + +#if 0 if (pRsp != NULL) { pRsp->ahandle = pRpc->ahandle; - tmsgSendRsp(pRsp); taosMemoryFree(pRsp); } else { if (code != 0 && terrno != 0) code = terrno; vmSendRsp(pVnode->pWrapper, pMsg, code); } +#endif } for (int32_t i = 0; i < numOfMsgs; i++) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2a9f540ffb..834d11fc20 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -49,7 +49,7 @@ void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodeClose(SVnode *pVnode); int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version); -int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg **pRsp); +int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); int vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index e7a3e1b29c..603b92d4a0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,7 +16,7 @@ #include "vnodeInt.h" static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq); -static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg **pRsp); +static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp); static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq); static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp); @@ -41,7 +41,7 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) { return 0; } -int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg **pRsp) { +int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { void *ptr = NULL; int ret; @@ -65,6 +65,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg ret = vnodeProcessCreateStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); return 0; case TDMT_VND_CREATE_TABLE: + pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; return vnodeProcessCreateTbReq(pVnode, pMsg, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pRsp); case TDMT_VND_ALTER_STB: return vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); @@ -74,14 +75,8 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg case TDMT_VND_DROP_TABLE: break; case TDMT_VND_SUBMIT: - /*printf("vnode %d write data %ld\n", TD_VID(pVnode), ver);*/ - if (pVnode->config.streamMode == 0) { - *pRsp = taosMemoryCalloc(1, sizeof(SRpcMsg)); - (*pRsp)->handle = pMsg->handle; - (*pRsp)->ahandle = pMsg->ahandle; - return vnodeProcessSubmitReq(pVnode, ptr, *pRsp); - } - break; + pRsp->msgType = TDMT_VND_SUBMIT_RSP; + return vnodeProcessSubmitReq(pVnode, ptr, pRsp); case TDMT_VND_MQ_SET_CONN: { if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { // TODO: handle error @@ -218,7 +213,7 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) { return 0; } -static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg **pRsp) { +static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg *pRsp) { SVCreateTbBatchReq vCreateTbBatchReq = {0}; SVCreateTbBatchRsp vCreateTbBatchRsp = {0}; tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq); @@ -270,12 +265,8 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR tSerializeSVCreateTbBatchRsp(msg, contLen, &vCreateTbBatchRsp); taosArrayDestroy(vCreateTbBatchRsp.rspList); - *pRsp = taosMemoryCalloc(1, sizeof(SRpcMsg)); - (*pRsp)->msgType = TDMT_VND_CREATE_TABLE_RSP; - (*pRsp)->pCont = msg; - (*pRsp)->contLen = contLen; - (*pRsp)->handle = pMsg->handle; - (*pRsp)->ahandle = pMsg->ahandle; + pRsp->pCont = msg; + pRsp->contLen = contLen; } return 0; @@ -308,7 +299,6 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg } // encode the response (TODO) - pRsp->msgType = TDMT_VND_SUBMIT_RSP; pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); pRsp->contLen = sizeof(SSubmitRsp); From 81f58bf74fe35b49824a5df3eee2cff68dfc1da7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 03:45:22 +0000 Subject: [PATCH 22/31] refactor: vnode --- include/common/tmsg.h | 1 - source/common/src/tmsg.c | 2 -- source/dnode/mnode/impl/src/mndStb.c | 1 - source/dnode/vnode/src/vnd/vnodeSvr.c | 11 +++++++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 67369bb42c..32d59c6929 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1431,7 +1431,6 @@ typedef struct { } SRSmaParam; typedef struct SVCreateTbReq { - int64_t ver; // use a general definition char* name; uint32_t ttl; uint32_t keep; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5c97743a62..30524471a9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -397,7 +397,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeString(buf, pReq->name); tlen += taosEncodeFixedU32(buf, pReq->ttl); tlen += taosEncodeFixedU32(buf, pReq->keep); @@ -465,7 +464,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { - buf = taosDecodeFixedI64(buf, &(pReq->ver)); buf = taosDecodeString(buf, &(pReq->name)); buf = taosDecodeFixedU32(buf, &(pReq->ttl)); buf = taosDecodeFixedU32(buf, &(pReq->keep)); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index fa24501651..3ead2f26a3 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -348,7 +348,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt tNameGetFullDbName(&name, dbFName); SVCreateTbReq req = {0}; - req.ver = 0; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 603b92d4a0..fface9d6c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -63,12 +63,14 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg switch (pMsg->msgType) { case TDMT_VND_CREATE_STB: ret = vnodeProcessCreateStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); - return 0; + break; case TDMT_VND_CREATE_TABLE: pRsp->msgType = TDMT_VND_CREATE_TABLE_RSP; - return vnodeProcessCreateTbReq(pVnode, pMsg, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pRsp); + vnodeProcessCreateTbReq(pVnode, pMsg, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pRsp); + break; case TDMT_VND_ALTER_STB: - return vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); + vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))); + break; case TDMT_VND_DROP_STB: vTrace("vgId:%d, process drop stb req", TD_VID(pVnode)); break; @@ -76,7 +78,8 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg break; case TDMT_VND_SUBMIT: pRsp->msgType = TDMT_VND_SUBMIT_RSP; - return vnodeProcessSubmitReq(pVnode, ptr, pRsp); + vnodeProcessSubmitReq(pVnode, ptr, pRsp); + break; case TDMT_VND_MQ_SET_CONN: { if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { // TODO: handle error From a036d2f3ff682705112f299034b483293fc38d52 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Apr 2022 13:07:23 +0800 Subject: [PATCH 23/31] fix(query): fix the bug caused by refactor in first/last function implementation. --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 17 ++++++------- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/executorimpl.c | 4 +-- source/libs/function/src/builtinsimpl.c | 25 +++++++++++++------ source/libs/scalar/src/scalar.c | 4 +-- .../libs/scalar/test/filter/filterTests.cpp | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 6 ++--- 9 files changed, 36 insertions(+), 28 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 15f3246013..043f2d1d12 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -196,7 +196,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f3bcf4bafc..975d097205 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1076,8 +1076,8 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { } } -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { - if (0 == numOfRows) { +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) { + if (0 == numOfRows || numOfRows <= existRows) { return TSDB_CODE_SUCCESS; } @@ -1088,19 +1088,16 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) } pColumn->varmeta.offset = (int32_t*)tmp; - memset(pColumn->varmeta.offset, 0, sizeof(int32_t) * numOfRows); - - pColumn->varmeta.length = 0; - pColumn->varmeta.allocLen = 0; - taosMemoryFreeClear(pColumn->pData); + memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows)); } else { char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + int32_t oldLen = BitmapLen(existRows); pColumn->nullbitmap = tmp; - memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); + memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); if (pColumn->info.type == TSDB_DATA_TYPE_NULL) { return TSDB_CODE_SUCCESS; @@ -1136,7 +1133,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = colInfoDataEnsureCapacity(p, numOfRows); + code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows); if (code) { return code; } @@ -1181,7 +1178,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - int32_t code = colInfoDataEnsureCapacity(pDst, pDataBlock->info.rows); + int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return NULL; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 9282f7197e..cf62ea8714 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -141,7 +141,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { colInfo.info.colId = pColSchema->colId; colInfo.info.type = pColSchema->type; - if (colInfoDataEnsureCapacity(&colInfo, numOfRows) < 0) { + if (colInfoDataEnsureCapacity(&colInfo, 0, numOfRows) < 0) { taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ab933f840a..2ad34d1561 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -392,7 +392,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - int32_t code = colInfoDataEnsureCapacity(&colInfo, pReadHandle->outputCapacity); + int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity); if (code != TSDB_CODE_SUCCESS) { goto _end; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index eaf09237de..366da58079 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -832,7 +832,7 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); - colInfoDataEnsureCapacity(pColData, 5); + colInfoDataEnsureCapacity(pColData, 0, 5); colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); @@ -1061,7 +1061,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } ASSERT(!IS_VAR_DATA_TYPE(type)); - colInfoDataEnsureCapacity(pColInfo, numOfRows); + colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { int64_t v = pFuncParam->param.i; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3b674759a3..5a32a6ffec 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -767,10 +767,18 @@ void percentileFinalize(SqlFunctionCtx* pCtx) { bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); - pEnv->calcMemSize = pNode->node.resType.bytes; + pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); return true; } +static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowIndex) { + if (pTsColInfo == NULL) { + return 0; + } + + return *(TSKEY*) colDataGetData(pTsColInfo, rowIndex); +} + // This ordinary first function does not care if current scan is ascending order or descending order scan // the OPTIMIZED version of first function will only handle the ascending order scan int32_t firstFunction(SqlFunctionCtx *pCtx) { @@ -792,8 +800,8 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet)? pInput->pColumnDataAgg[0]:NULL; - TSKEY startKey = *(int64_t*)(pInput->pPTS ? colDataGetData(pInput->pPTS, 0) : 0); - TSKEY endKey = *(int64_t*)(pInput->pPTS ? colDataGetData(pInput->pPTS, pInput->totalRows - 1) : 0); + TSKEY startKey = getRowPTs(pInput->pPTS, 0); + TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1); int32_t blockDataOrder = (startKey <= endKey)? TSDB_ORDER_ASC:TSDB_ORDER_DESC; @@ -812,12 +820,14 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { } char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); - TSKEY cts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { + if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { memcpy(buf, data, bytes); *(TSKEY*)(buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + + pResInfo->numOfRes = 1; } numOfElems++; @@ -838,12 +848,13 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { } char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); - TSKEY cts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { + if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { memcpy(buf, data, bytes); *(TSKEY*)(buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + pResInfo->numOfRes = 1; } numOfElems++; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0432ae1df8..337a773d5c 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -30,7 +30,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { pColumnData->info.scale = pType->scale; pColumnData->info.precision = pType->precision; - int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); + int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); @@ -45,7 +45,7 @@ int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); - colInfoDataEnsureCapacity(out->columnData, 1); + colInfoDataEnsureCapacity(out->columnData, 0, 1); int32_t code = vectorConvertImpl(&in, out); sclFreeParam(&in); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 26ef5dbd44..42998aba00 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -155,7 +155,7 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - colInfoDataEnsureCapacity(pColumn, rowNum); + colInfoDataEnsureCapacity(pColumn, 0, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 61ef2fdce2..09d6528dd8 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -99,7 +99,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s SColumnInfoData idata = {0}; idata.info = *colInfo; - colInfoDataEnsureCapacity(&idata, rows); + colInfoDataEnsureCapacity(&idata, 0, rows); taosArrayPush(res->pDataBlock, &idata); @@ -186,7 +186,7 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in res->info.numOfCols++; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - colInfoDataEnsureCapacity(pColumn, rowNum); + colInfoDataEnsureCapacity(pColumn, 0, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -1467,7 +1467,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t input->numOfRows = num; input->columnData->info = createColumnInfo(0, type, bytes); - colInfoDataEnsureCapacity(input->columnData, num); + colInfoDataEnsureCapacity(input->columnData, 0, num); if (setVal) { for (int32_t i = 0; i < num; ++i) { From a70181194b709dfd8ef7ea9f943f2cf174719171 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 19 Apr 2022 13:15:44 +0800 Subject: [PATCH 24/31] refactor(query): refactor string functions TD-14836 TD-14833 --- source/libs/scalar/src/sclfunc.c | 83 ++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e7ff0bde91..02c6006f60 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -296,7 +296,6 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *in = pInputData->pData + pInputData->varmeta.offset[0]; int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { @@ -305,8 +304,8 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP continue; } + char *in = pInputData->pData + pInputData->varmeta.offset[i]; out[i] = lenFn(in, type); - in += varDataTLen(in); } pOutput->numOfRows = pInput->numOfRows; @@ -316,7 +315,8 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, int32_t type, int16_t *dataLen) { if (hasNchar && type == TSDB_DATA_TYPE_VARCHAR) { TdUcs4 *newBuf = taosMemoryCalloc((varDataLen(input) + 1) * TSDB_NCHAR_SIZE, 1); - bool ret = taosMbsToUcs4(varDataVal(input), varDataLen(input), newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, NULL); + int32_t len = varDataLen(input); + bool ret = taosMbsToUcs4(varDataVal(input), len, newBuf, (varDataLen(input) + 1) * TSDB_NCHAR_SIZE, &len); if (!ret) { taosMemoryFree(newBuf); return TSDB_CODE_FAILED; @@ -345,6 +345,7 @@ static int32_t getNumOfNullEntries(SColumnInfoData *pColumnInfoData, int32_t num } int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t ret = TSDB_CODE_SUCCESS; SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); @@ -360,7 +361,6 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; - input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; int32_t factor = 1; if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; @@ -378,8 +378,8 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu outputBuf = taosMemoryCalloc(outputLen, 1); char *output = outputBuf; - bool hasNull = false; for (int32_t k = 0; k < numOfRows; ++k) { + bool hasNull = false; for (int32_t i = 0; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k)) { colDataAppendNULL(pOutputData, k); @@ -392,14 +392,18 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu continue; } + int16_t dataLen = 0; for (int32_t i = 0; i < inputNum; ++i) { - int32_t ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if (pInput[i].numOfRows == 1) { + input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; + } else { + input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[k]; } - if (pInput[i].numOfRows != 1) { - input[i] += varDataTLen(input[i]); + + ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); + if (ret != TSDB_CODE_SUCCESS) { + goto DONE; } } varDataSetLen(output, dataLen); @@ -408,15 +412,18 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } pOutput->numOfRows = numOfRows; + +DONE: taosMemoryFree(input); taosMemoryFree(outputBuf); taosMemoryFree(pInputData); - return TSDB_CODE_SUCCESS; + return ret; } int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + int32_t ret = TSDB_CODE_SUCCESS; SColumnInfoData **pInputData = taosMemoryCalloc(inputNum, sizeof(SColumnInfoData *)); SColumnInfoData *pOutputData = pOutput->columnData; char **input = taosMemoryCalloc(inputNum, POINTER_BYTES); @@ -432,7 +439,6 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } for (int32_t i = 0; i < inputNum; ++i) { pInputData[i] = pInput[i].columnData; - input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; int32_t factor = 1; if (hasNchar && (GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_VARCHAR)) { factor = TSDB_NCHAR_SIZE; @@ -460,40 +466,53 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } int16_t dataLen = 0; + bool hasNull = false; for (int32_t i = 1; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k)) { - continue; + hasNull = true; + break; } - int32_t ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); + if (pInput[i].numOfRows == 1) { + input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[0]; + } else { + input[i] = pInputData[i]->pData + pInputData[i]->varmeta.offset[k]; + } + + ret = concatCopyHelper(input[i], output, hasNchar, GET_PARAM_TYPE(&pInput[i]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { - return ret; + goto DONE; } - if (pInput[i].numOfRows != 1) { - input[i] += varDataTLen(input[i]); - } if (i < inputNum - 1) { //insert the separator - char *sep = pInputData[0]->pData; - int32_t ret = concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen); + char *sep = (pInput[0].numOfRows == 1) ? colDataGetData(pInputData[0], 0) : colDataGetData(pInputData[0], k); + ret = concatCopyHelper(sep, output, hasNchar, GET_PARAM_TYPE(&pInput[0]), &dataLen); if (ret != TSDB_CODE_SUCCESS) { - return ret; + goto DONE; } } } - varDataSetLen(output, dataLen); - colDataAppend(pOutputData, k, output, false); - output += varDataTLen(output); + + if (hasNull) { + colDataAppendNULL(pOutputData, k); + memset(output, 0, dataLen); + } else { + varDataSetLen(output, dataLen); + colDataAppend(pOutputData, k, output, false); + output += varDataTLen(output); + } } pOutput->numOfRows = numOfRows; + +DONE: taosMemoryFree(input); taosMemoryFree(outputBuf); taosMemoryFree(pInputData); - return TSDB_CODE_SUCCESS; + return ret; } static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput, _conv_fn convFn) { @@ -505,12 +524,9 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *input = pInputData->pData + pInputData->varmeta.offset[0]; - char *output = NULL; - int32_t outputLen = pInputData->varmeta.length; char *outputBuf = taosMemoryCalloc(outputLen, 1); - output = outputBuf; + char *output = outputBuf; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { @@ -518,6 +534,7 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala continue; } + char *input = pInputData->pData + pInputData->varmeta.offset[i]; int32_t len = varDataLen(input); if (type == TSDB_DATA_TYPE_VARCHAR) { for (int32_t j = 0; j < len; ++j) { @@ -530,7 +547,6 @@ static int32_t doCaseConvFunction(SScalarParam *pInput, int32_t inputNum, SScala } varDataSetLen(output, len); colDataAppend(pOutputData, i, output, false); - input += varDataTLen(input); output += varDataTLen(output); } @@ -550,18 +566,16 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar SColumnInfoData *pInputData = pInput->columnData; SColumnInfoData *pOutputData = pOutput->columnData; - char *input = pInputData->pData + pInputData->varmeta.offset[0]; - char *output = NULL; - int32_t outputLen = pInputData->varmeta.length; char *outputBuf = taosMemoryCalloc(outputLen, 1); - output = outputBuf; + char *output = outputBuf; for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { colDataAppendNULL(pOutputData, i); continue; } + char *input = pInputData->pData + pInputData->varmeta.offset[i]; int32_t len = varDataLen(input); int32_t charLen = (type == TSDB_DATA_TYPE_VARCHAR) ? len : len / TSDB_NCHAR_SIZE; @@ -569,7 +583,6 @@ static int32_t doTrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarPar varDataSetLen(output, len); colDataAppend(pOutputData, i, output, false); - input += varDataTLen(input); output += varDataTLen(output); } From 2d7cf35fb03dce44165261f152d2502f0096e49e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 19 Apr 2022 13:25:19 +0800 Subject: [PATCH 25/31] fix: some syntax parsing problems --- include/util/tdef.h | 2 +- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/parTranslater.c | 41 +++++++++++--------- source/libs/parser/src/sql.c | 2 +- source/libs/planner/src/planOptimizer.c | 4 ++ source/libs/planner/test/planTestUtil.cpp | 2 + source/libs/planner/test/planTestUtil.h | 2 + source/libs/planner/test/plannerTestMain.cpp | 20 ++++++++++ 8 files changed, 54 insertions(+), 21 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 5b863f5920..ec90dd888a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -199,7 +199,7 @@ typedef enum EOperatorType { } EOperatorType; typedef enum ELogicConditionType { - LOGIC_COND_TYPE_AND, + LOGIC_COND_TYPE_AND = 1, LOGIC_COND_TYPE_OR, LOGIC_COND_TYPE_NOT, } ELogicConditionType; diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 33b525cf3a..93e56424df 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -646,7 +646,7 @@ predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, D); - A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); } predicate(A) ::= expression(B) IS NULL(C). { SToken s = getTokenFromRawExprNode(pCxt, B); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a7e7e01948..925be3bd82 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -251,6 +251,9 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol pCol->colType = pProjCol->colType; } strcpy(pCol->colName, pExpr->aliasName); + if ('\0' == pCol->node.aliasName[0]) { + strcpy(pCol->node.aliasName, pCol->colName); + } pCol->node.resType = pExpr->resType; } @@ -381,23 +384,7 @@ static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { } res = (found ? DEAL_RES_CONTINUE : translateColumnWithoutPrefix(pCxt, pCol)); } - - if (DEAL_RES_ERROR == res) { - return res; - } - - if (SQL_CLAUSE_WINDOW == pCxt->currClause && QUERY_NODE_STATE_WINDOW == nodeType(pCxt->pCurrStmt->pWindow)) { - if (!IS_INTEGER_TYPE(pCol->node.resType.type)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); - } - if (COLUMN_TYPE_TAG == pCol->colType) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); - } - if (TSDB_SUPER_TABLE == pCol->tableType) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE); - } - } - return DEAL_RES_CONTINUE; + return res; } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { @@ -1200,9 +1187,27 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* return TSDB_CODE_SUCCESS; } +static EDealRes checkStateExpr(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + STranslateContext* pCxt = pContext; + SColumnNode* pCol = (SColumnNode*)pNode; + if (!IS_INTEGER_TYPE(pCol->node.resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); + } + if (COLUMN_TYPE_TAG == pCol->colType) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); + } + if (TSDB_SUPER_TABLE == pCol->tableType) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE); + } + } + return DEAL_RES_CONTINUE; +} + static int32_t checkStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { + nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); // todo check for "function not support for state_window" - return TSDB_CODE_SUCCESS; + return pCxt->errCode; } static int32_t checkSessionWindow(STranslateContext* pCxt, SSessionWindowNode* pSession) { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 224d647017..110f4f9fb5 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -3730,7 +3730,7 @@ static YYACTIONTYPE yy_reduce( { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy456); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } yymsp[-5].minor.yy456 = yylhsminor.yy456; break; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 6fcfc7df07..e796d126eb 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -77,6 +77,10 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { return false; } + // todo: release after function splitting + if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType) { + return false; + } if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) && QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent))) { return false; diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index e4c1d41000..25457d3e41 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -32,6 +32,8 @@ using namespace testing; } \ } while(0); +bool g_isDump = false; + class PlannerTestBaseImpl { public: void useDb(const string& acctId, const string& db) { diff --git a/source/libs/planner/test/planTestUtil.h b/source/libs/planner/test/planTestUtil.h index 71039082f9..dbd14237ee 100644 --- a/source/libs/planner/test/planTestUtil.h +++ b/source/libs/planner/test/planTestUtil.h @@ -32,4 +32,6 @@ private: std::unique_ptr impl_; }; +extern bool g_isDump; + #endif // PLAN_TEST_UTIL_H diff --git a/source/libs/planner/test/plannerTestMain.cpp b/source/libs/planner/test/plannerTestMain.cpp index 36a46ab99e..2878dc7954 100644 --- a/source/libs/planner/test/plannerTestMain.cpp +++ b/source/libs/planner/test/plannerTestMain.cpp @@ -18,6 +18,7 @@ #include #include "mockCatalog.h" +#include "planTestUtil.h" class PlannerEnv : public testing::Environment { public: @@ -34,8 +35,27 @@ public: virtual ~PlannerEnv() {} }; +static void parseArg(int argc, char* argv[]) { + int opt = 0; + const char *optstring = ""; + static struct option long_options[] = { + {"dump", no_argument, NULL, 'd'}, + {0, 0, 0, 0} + }; + while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) { + switch (opt) { + case 'd': + g_isDump = true; + break; + default: + break; + } + } +} + int main(int argc, char* argv[]) { testing::AddGlobalTestEnvironment(new PlannerEnv()); testing::InitGoogleTest(&argc, argv); + parseArg(argc, argv); return RUN_ALL_TESTS(); } From 0d89d9378759f34a09ec35b63f7d19f2895dead2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 19 Apr 2022 05:42:16 +0000 Subject: [PATCH 26/31] refactor: vnode --- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/vnd/vnodeWrite.c | 18 ------------------ 2 files changed, 19 deletions(-) delete mode 100644 source/dnode/vnode/src/vnd/vnodeWrite.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 5a6fdd1b40..6a318b8e10 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -13,7 +13,6 @@ target_sources( "src/vnd/vnodeInt.c" "src/vnd/vnodeQuery.c" "src/vnd/vnodeStateMgr.c" - "src/vnd/vnodeWrite.c" "src/vnd/vnodeModule.c" "src/vnd/vnodeSvr.c" diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c deleted file mode 100644 index 16b881f00d..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeInt.h" - -/* ------------------------ STATIC METHODS ------------------------ */ From 0787021ca0f786f2fd9696bfd893c36eb285cf3a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Apr 2022 13:47:40 +0800 Subject: [PATCH 27/31] fix(query): check the primary timestamp null case and report error with inforamtion to caller. --- source/libs/parser/src/parInsert.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index acc597d61b..d774a2d024 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -460,8 +460,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int if (isNullStr(pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { - int64_t tmpVal = 0; - return func(pMsgBuf, &tmpVal, pSchema->bytes, param); + return buildSyntaxErrMsg(pMsgBuf, "primary timestamp can not be null", pToken->z); } return func(pMsgBuf, NULL, 0, param); From 8959e4437b7c011224379df1c82039ad98fabb76 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Tue, 19 Apr 2022 14:13:34 +0800 Subject: [PATCH 28/31] trow refactor --- include/common/trow.h | 50 +++++++++++++------------- source/libs/parser/inc/parInsertData.h | 4 +-- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index dbbdef84a0..963542fb31 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -308,21 +308,21 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, // use literal value directly and not use formula to simplify the codes switch (nOffset) { case 0: - // *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); + *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); // set the value and clear other partitions for offset 0 - *pDestByte = (valType << 6); + // *pDestByte |= (valType << 6); break; case 1: - // *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); - *pDestByte |= (valType << 4); + *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); + // *pDestByte |= (valType << 4); break; case 2: - // *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); - *pDestByte |= (valType << 2); + *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); + // *pDestByte |= (valType << 2); break; case 3: - // *pDestByte = ((*pDestByte) & 0xFC) | valType; - *pDestByte |= (valType); + *pDestByte = ((*pDestByte) & 0xFC) | valType; + // *pDestByte |= (valType); break; default: TASSERT(0); @@ -417,37 +417,37 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T // use literal value directly and not use formula to simplify the codes switch (nOffset) { case 0: - // *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); + *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); // set the value and clear other partitions for offset 0 - *pDestByte = (valType << 7); + // *pDestByte |= (valType << 7); break; case 1: - // *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); - *pDestByte |= (valType << 6); + *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); + // *pDestByte |= (valType << 6); break; case 2: - // *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); - *pDestByte |= (valType << 5); + *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); + // *pDestByte |= (valType << 5); break; case 3: - // *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); - *pDestByte |= (valType << 4); + *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); + // *pDestByte |= (valType << 4); break; case 4: - // *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); - *pDestByte |= (valType << 3); + *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); + // *pDestByte |= (valType << 3); break; case 5: - // *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); - *pDestByte |= (valType << 2); + *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); + // *pDestByte |= (valType << 2); break; case 6: - // *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); - *pDestByte |= (valType << 1); + *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); + // *pDestByte |= (valType << 1); break; case 7: - // *pDestByte = ((*pDestByte) & 0xFE) | valType; - *pDestByte |= (valType); + *pDestByte = ((*pDestByte) & 0xFE) | valType; + // *pDestByte |= (valType); break; default: TASSERT(0); @@ -656,6 +656,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { case TD_ROW_TP: #ifdef TD_SUPPORT_BITMAP pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif // the primary TS key is stored separatedly len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; @@ -665,6 +666,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { case TD_ROW_KV: #ifdef TD_SUPPORT_BITMAP pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); #endif len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + pBuilder->nBoundBitmaps; // add diff --git a/source/libs/parser/inc/parInsertData.h b/source/libs/parser/inc/parInsertData.h index ee17de50e0..cf87f57d4f 100644 --- a/source/libs/parser/inc/parInsertData.h +++ b/source/libs/parser/inc/parInsertData.h @@ -98,7 +98,7 @@ static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart *colIdx = schemaIdx; } else { - *toffset = idx * sizeof(SColIdx); // the offset of SColIdx + *toffset = idx * sizeof(SKvRowIdx); // the offset of SKvRowIdx *colIdx = idx; } } else { @@ -108,7 +108,7 @@ static FORCE_INLINE void getSTSRowAppendInfo(SSchema *pSchema, uint8_t rowType, *toffset = (spd->cols + schemaIdx)->toffset; *colIdx = schemaIdx; } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx); + *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx); *colIdx = (spd->colIdxInfo + idx)->finalIdx; } } From 556f6a96ea96eb3fec18fd8918357f9bf6d774d7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 19 Apr 2022 15:46:11 +0800 Subject: [PATCH 29/31] feat: delimit the time range according to the filtering conditions --- source/libs/planner/CMakeLists.txt | 2 +- source/libs/planner/src/planOptimizer.c | 235 +++++++++++++++++------- 2 files changed, 174 insertions(+), 63 deletions(-) diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index a095a6205f..f0bf32bf17 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util nodes catalog cjson parser function qcom + PRIVATE os util nodes catalog cjson parser function qcom scalar PUBLIC transport ) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index e796d126eb..34bee3fd0b 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -15,6 +15,7 @@ #include "planInt.h" #include "functionMgt.h" +#include "filter.h" #define OPTIMIZE_FLAG_MASK(n) (1 << n) @@ -195,11 +196,175 @@ static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { return code; } -static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - // todo +static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { + SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->condType = LOGIC_COND_TYPE_AND; + int32_t code = nodesListMakeAppend(&pLogicCond->pParameterList, *pSrc); + if (TSDB_CODE_SUCCESS == code) { + *pSrc = NULL; + code = nodesListMakeAppend(&pLogicCond->pParameterList, *pDst); + } + if (TSDB_CODE_SUCCESS == code) { + *pDst = (SNode*)pLogicCond; + } else { + nodesDestroyNode(pLogicCond); + } + return code; +} + +static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) { + if (NULL == *pSrc) { + return TSDB_CODE_SUCCESS; + } + + if (1 == LIST_LENGTH(*pSrc)) { + *pDst = nodesListGetNode(*pSrc, 0); + nodesClearList(*pSrc); + } else { + SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->condType = LOGIC_COND_TYPE_AND; + pLogicCond->pParameterList = *pSrc; + *pDst = (SNode*)pLogicCond; + } + *pSrc = NULL; + return TSDB_CODE_SUCCESS; } +static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { + if (NULL == *pCond) { + TSWAP(*pCond, *pAdditionalCond, SNode*); + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond)) { + code = nodesListAppend(((SLogicConditionNode*)*pCond)->pParameterList, *pAdditionalCond); + if (TSDB_CODE_SUCCESS == code) { + *pAdditionalCond = NULL; + } + } else { + code = cpdMergeCond(pCond, pAdditionalCond); + } + return code; +} + +static EDealRes cpdIsPrimaryKeyCondImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + *((bool*)pContext) = ((PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) ? true : false); + return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static bool cpdIsPrimaryKeyCond(SNode* pNode) { + bool isPrimaryKeyCond = false; + nodesWalkExpr(pNode, cpdIsPrimaryKeyCondImpl, &isPrimaryKeyCond); + return isPrimaryKeyCond; +} + +static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pScan->node.pConditions; + + int32_t code = TSDB_CODE_SUCCESS; + + SNodeList* pPrimaryKeyConds = NULL; + SNodeList* pOtherConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + if (cpdIsPrimaryKeyCond(pCond)) { + code = nodesListMakeAppend(&pPrimaryKeyConds, nodesCloneNode(pCond)); + } else { + code = nodesListMakeAppend(&pOtherConds, nodesCloneNode(pCond)); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNode* pTempPrimaryKeyCond = NULL; + SNode* pTempOtherCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = cpdMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds); + } + if (TSDB_CODE_SUCCESS == code) { + code = cpdMergeConds(&pTempOtherCond, &pOtherConds); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPrimaryKeyCond = pTempPrimaryKeyCond; + *pOtherCond = pTempOtherCond; + nodesDestroyNode(pScan->node.pConditions); + pScan->node.pConditions = NULL; + } else { + nodesDestroyList(pPrimaryKeyConds); + nodesDestroyList(pOtherConds); + nodesDestroyNode(pTempPrimaryKeyCond); + nodesDestroyNode(pTempOtherCond); + } + + return code; +} + +static int32_t cpdPartitionScanCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pScan->node.pConditions) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pScan->node.pConditions)->condType) { + return cpdPartitionScanLogicCond(pScan, pPrimaryKeyCond, pOtherCond); + } + + if (cpdIsPrimaryKeyCond(pScan->node.pConditions)) { + *pPrimaryKeyCond = pScan->node.pConditions; + } else { + *pOtherCond = pScan->node.pConditions; + } + pScan->node.pConditions = NULL; + + return TSDB_CODE_SUCCESS; +} + +static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { + bool isStrict = false; + int32_t code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); + if (TSDB_CODE_SUCCESS == code) { + if (isStrict) { + nodesDestroyNode(*pPrimaryKeyCond); + } else { + code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + } + *pPrimaryKeyCond = NULL; + } + return code; +} + +static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { + if (NULL == pScan->node.pConditions) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPrimaryKeyCond = NULL; + SNode* pOtherCond = NULL; + int32_t code = cpdPartitionScanCond(pScan, &pPrimaryKeyCond, &pOtherCond); + if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { + code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond); + } + if (TSDB_CODE_SUCCESS == code) { + pScan->node.pConditions = pOtherCond; + } + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pPrimaryKeyCond); + nodesDestroyNode(pOtherCond); + } + + return code; +} + static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) { SNode* pTableCol = NULL; FOREACH(pTableCol, pTableCols) { @@ -230,28 +395,6 @@ static ECondAction cpdCondAction(EJoinType joinType, SNodeList* pLeftCols, SNode (cxt.havaLeftCol && cxt.haveRightCol ? COND_ACTION_PUSH_JOIN : (cxt.havaLeftCol ? COND_ACTION_PUSH_LEFT_CHILD : COND_ACTION_PUSH_RIGHT_CHILD))); } -static int32_t cpdMakeCond(SNodeList** pConds, SNode** pCond) { - if (NULL == *pConds) { - return TSDB_CODE_SUCCESS; - } - - if (1 == LIST_LENGTH(*pConds)) { - *pCond = nodesListGetNode(*pConds, 0); - nodesClearList(*pConds); - } else { - SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); - if (NULL == pLogicCond) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pLogicCond->condType = LOGIC_COND_TYPE_AND; - pLogicCond->pParameterList = *pConds; - *pCond = (SNode*)pLogicCond; - } - *pConds = NULL; - - return TSDB_CODE_SUCCESS; -} - static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** pLeftChildCond, SNode** pRightChildCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pJoin->node.pConditions; if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { @@ -288,16 +431,16 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo SNode* pTempRightChildCond = NULL; SNode* pTempRemainCond = NULL; if (TSDB_CODE_SUCCESS == code) { - code = cpdMakeCond(&pOnConds, &pTempOnCond); + code = cpdMergeConds(&pTempOnCond, &pOnConds); } if (TSDB_CODE_SUCCESS == code) { - code = cpdMakeCond(&pLeftChildConds, &pTempLeftChildCond); + code = cpdMergeConds(&pTempLeftChildCond, &pLeftChildConds); } if (TSDB_CODE_SUCCESS == code) { - code = cpdMakeCond(&pRightChildConds, &pTempRightChildCond); + code = cpdMergeConds(&pTempRightChildCond, &pRightChildConds); } if (TSDB_CODE_SUCCESS == code) { - code = cpdMakeCond(&pRemainConds, &pTempRemainCond); + code = cpdMergeConds(&pTempRemainCond, &pRemainConds); } if (TSDB_CODE_SUCCESS == code) { @@ -345,44 +488,12 @@ static int32_t cpdPartitionCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNode** } } -static int32_t cpdCondAppend(SOptimizeContext* pCxt, SNode** pCond, SNode** pAdditionalCond) { - if (NULL == *pCond) { - TSWAP(*pCond, *pAdditionalCond, SNode*); - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCond)) { - code = nodesListAppend(((SLogicConditionNode*)*pCond)->pParameterList, *pAdditionalCond); - if (TSDB_CODE_SUCCESS == code) { - *pAdditionalCond = NULL; - } - } else { - SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); - if (NULL == pLogicCond) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pLogicCond->condType = LOGIC_COND_TYPE_AND; - code = nodesListMakeAppend(&pLogicCond->pParameterList, *pAdditionalCond); - if (TSDB_CODE_SUCCESS == code) { - *pAdditionalCond = NULL; - code = nodesListMakeAppend(&pLogicCond->pParameterList, *pCond); - } - if (TSDB_CODE_SUCCESS == code) { - *pCond = (SNode*)pLogicCond; - } else { - nodesDestroyNode(pLogicCond); - } - } - return code; -} - static int32_t cpdPushCondToOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode** pCond) { - return cpdCondAppend(pCxt, &pJoin->pOnConditions, pCond); + return cpdCondAppend(&pJoin->pOnConditions, pCond); } static int32_t cpdPushCondToScan(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pCond) { - return cpdCondAppend(pCxt, &pScan->node.pConditions, pCond); + return cpdCondAppend(&pScan->node.pConditions, pCond); } static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SNode** pCond) { From 99c36d6e8246d4572c958b1a8a311b5228016305 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 19 Apr 2022 16:02:46 +0800 Subject: [PATCH 30/31] [test: add sync test cases] --- .../script/tsim/sync/insertDataByRunBack.sim | 63 +++ tests/script/tsim/sync/oneReplica1VgElect.sim | 363 ++++++++++++++ .../sync/oneReplica1VgElectWithInsert.sim | 401 +++++++++++++++ .../script/tsim/sync/threeReplica1VgElect.sim | 391 +++++++++++++++ .../sync/threeReplica1VgElectWihtInsert.sim | 469 ++++++++++++++++++ 5 files changed, 1687 insertions(+) create mode 100644 tests/script/tsim/sync/insertDataByRunBack.sim create mode 100644 tests/script/tsim/sync/oneReplica1VgElect.sim create mode 100644 tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim create mode 100644 tests/script/tsim/sync/threeReplica1VgElect.sim create mode 100644 tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim diff --git a/tests/script/tsim/sync/insertDataByRunBack.sim b/tests/script/tsim/sync/insertDataByRunBack.sim new file mode 100644 index 0000000000..c86cd3844b --- /dev/null +++ b/tests/script/tsim/sync/insertDataByRunBack.sim @@ -0,0 +1,63 @@ + +sql connect + +print ================ insert data +$dbNamme = db +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$rowNum = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +sql use $dbNamme + +$loop_cnt = 0 +loop_insert: +print ====> loop insert data, but once check if there is stop insert flag in interaction talbe from main thread +sql select * from interaction +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] +if $rows == 2 then + if $data[1][1] == stop then + goto end_insert + endi +endi + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +if $loop_cnt == 0 then + print ====> notify main to working for insert data + sql insert into interaction values (now, 'working', 0, 0); +endi +$loop_cnt = $loop_cnt + 1 +goto loop_insert + +end_insert: +print ====> save insert data rows to main thread + +$totalRowsOfCtb = $rowNum * $loop_cnt +$totalRowsOfStb = $totalRowsOfCtb * $tbNum +sql insert into interaction values (now, 'end', $totalRowsOfCtb, $totalRowsOfStb ); +print ====> totalRowsOfCtb: $totalRowsOfCtb , totalRowsOfStb: $totalRowsOfStb + + + diff --git a/tests/script/tsim/sync/oneReplica1VgElect.sim b/tests/script/tsim/sync/oneReplica1VgElect.sim new file mode 100644 index 0000000000..9726c0fdf5 --- /dev/null +++ b/tests/script/tsim/sync/oneReplica1VgElect.sim @@ -0,0 +1,363 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][0] != 1 then + return -1 +endi +if $data[0][4] != ready then + goto check_dnode_ready +endi + +#sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +check_dnode_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> dnodes not ready! + return -1 +endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][4] != ready then + goto check_dnode_ready_1 +endi +if $data[1][4] != ready then + goto check_dnode_ready_1 +endi +if $data[2][4] != ready then + goto check_dnode_ready_1 +endi +if $data[3][4] != ready then + goto check_dnode_ready_1 +endi + +$vgroups = 1 +$replica = 1 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $rows != 3 then + return -1 +endi +if $data(db)[20] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != NULL then + goto check_vg_ready + endi + if $data[0][8] != NULL then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + goto vg_ready +endi +if $data[0][6] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready + endi + if $data[0][8] != NULL then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + goto vg_ready +endi +if $data[0][8] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready + endi + if $data[0][6] != NULL then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + goto vg_ready +endi +vg_ready: + +print ====> create stable/child table, insert data, and select +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$rowNum = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + print ====> insert rows: $rowNum into $ctb and $ntb + + $i = $i + 1 + $tstart = 1640966400000 +endw + +$totalTblNum = $tbNum * 2 +sql show tables +if $rows != $totalTblNum then + return -1 +endi + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +$totalRowsOfStb = $rowNum * $tbNum +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + $dnodeId = $data[0][3] +elif $data[0][6] == LEADER then + $dnodeId = $data[0][5] +endi +elif $data[0][8] == LEADER then + $dnodeId = $data[0][7] +else + print ====> no leader vnode!!! + return -1 +endi + +$dnodeId = dnode . $dnodeId +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + +$loop_cnt = 0 +check_vg_ready_2: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups switch fail!!! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != NULL then + goto check_vg_ready_2 + endi + if $data[0][8] != NULL then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] + goto vg_ready_2 +endi +if $data[0][6] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_2 + endi + if $data[0][8] != NULL then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] + goto vg_ready_2 +endi +if $data[0][8] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_2 + endi + if $data[0][6] != NULL then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] + goto vg_ready_2 +endi +vg_ready_2: + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> stop and start all dnode(not include the dnode where mnode is located), then query +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode2 -s start + +$loop_cnt = 0 +check_vg_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> after restart dnode, vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != NULL then + goto check_vg_ready_1 + endi + if $data[0][8] != NULL then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][6] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_1 + endi + if $data[0][8] != NULL then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][8] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_1 + endi + if $data[0][6] != NULL then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +vg_ready_1: + +print ====> after restart dnode2/dnode3/dnode4, query stb/ntb count(*) +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim new file mode 100644 index 0000000000..972447ba9b --- /dev/null +++ b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim @@ -0,0 +1,401 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][0] != 1 then + return -1 +endi +if $data[0][4] != ready then + goto check_dnode_ready +endi + +#sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +check_dnode_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> dnodes not ready! + return -1 +endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][4] != ready then + goto check_dnode_ready_1 +endi +if $data[1][4] != ready then + goto check_dnode_ready_1 +endi +if $data[2][4] != ready then + goto check_dnode_ready_1 +endi +if $data[3][4] != ready then + goto check_dnode_ready_1 +endi + +$vgroups = 1 +$replica = 1 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $rows != 3 then + return -1 +endi +if $data(db)[20] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != NULL then + goto check_vg_ready + endi + if $data[0][8] != NULL then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + goto vg_ready +endi +if $data[0][6] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready + endi + if $data[0][8] != NULL then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + goto vg_ready +endi +if $data[0][8] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready + endi + if $data[0][6] != NULL then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + goto vg_ready +endi +vg_ready: + +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +#$rowNum = 10 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +# $x = 0 +# while $x < $rowNum +# $binary = ' . binary +# $binary = $binary . $i +# $binary = $binary . ' +# +# sql insert into $ctb values ($tstart , $i , $x , $binary ) +# sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw + +# print ====> insert rows: $rowNum into $ctb and $ntb + + $i = $i + 1 +# $tstart = 1640966400000 +endw + +$totalTblNum = $tbNum * 2 +sql show tables +if $rows != $totalTblNum then + return -1 +endi + +print ====> create a normal table for interaction between main and back threads +sql create table interaction (ts timestamp, flag binary(10), childrows int, stbrows int) + +print ====> start to run_back to insert data +run_back tsim/tmq/insertDataByRunBack.sim + + +print ====> waiting insert thread starting insert data +waiting_insert_data: +sql select * from interaction +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] +if $rows == 1 then + if $data[0][1] == working then + goto start_switch_leader + endi +endi +goto waiting_insert_data + +start_switch_leader: + +$switch_loop_cnt = 0 +switch_leader_loop: + +print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + $dnodeId = $data[0][3] +elif $data[0][6] == LEADER then + $dnodeId = $data[0][5] +endi +elif $data[0][8] == LEADER then + $dnodeId = $data[0][7] +else + print ====> no leader vnode!!! + return -1 +endi + +$dnodeId = dnode . $dnodeId +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + +$loop_cnt = 0 +check_vg_ready_2: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups switch fail!!! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != NULL then + goto check_vg_ready_2 + endi + if $data[0][8] != NULL then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] + goto vg_ready_2 +endi +if $data[0][6] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_2 + endi + if $data[0][8] != NULL then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] + goto vg_ready_2 +endi +if $data[0][8] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_2 + endi + if $data[0][6] != NULL then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] + goto vg_ready_2 +endi +vg_ready_2: + +$switch_loop_cnt = $switch_loop_cnt + 1 +if $switch_loop_cnt < 3 then + print ====> start $dnodeId + system sh/exec.sh -n $dnodeId -s start + goto switch_leader_loop +endi + +loop_switch_end: + +print ====> notify insert thread to stop insert data +sql insert into interaction values (now, 'stop', 0, 0); + +print ====> waiting insert thread to stop insert data +waiting_stop_data: +sql select * from interaction +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $rows == 3 then + if $data[2][1] == end then + $totalRowsOfCtb = $data[2][2] + $totalRowsOfStb = $data[2][3] + goto check_affected_rows + endi +endi +goto waiting_stop_data + +check_affected_rows: + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> stop and start all dnode(not include the dnode where mnode is located), then query +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode2 -s start + +$loop_cnt = 0 +check_vg_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> after restart dnode, vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != NULL then + goto check_vg_ready_1 + endi + if $data[0][8] != NULL then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][6] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_1 + endi + if $data[0][8] != NULL then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][8] == LEADER then + if $data[0][4] != NULL then + goto check_vg_ready_1 + endi + if $data[0][6] != NULL then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +vg_ready_1: + +print ====> after restart dnode2/dnode3/dnode4, query stb/ntb count(*) +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim new file mode 100644 index 0000000000..f709c72dd7 --- /dev/null +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -0,0 +1,391 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][0] != 1 then + return -1 +endi +if $data[0][4] != ready then + goto check_dnode_ready +endi + +#sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +check_dnode_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> dnodes not ready! + return -1 +endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][4] != ready then + goto check_dnode_ready_1 +endi +if $data[1][4] != ready then + goto check_dnode_ready_1 +endi +if $data[2][4] != ready then + goto check_dnode_ready_1 +endi +if $data[3][4] != ready then + goto check_dnode_ready_1 +endi + +$vgroups = 1 +$replica = 3 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $rows != 3 then + return -1 +endi +if $data(db)[20] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != FLLOWER then + goto check_vg_ready + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + goto vg_ready +endi +if $data[0][6] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + goto vg_ready +endi +if $data[0][8] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready + endi + if $data[0][6] != FLLOWER then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + goto vg_ready +endi +vg_ready: + +print ====> create stable/child table, insert data, and select +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$rowNum = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + print ====> insert rows: $rowNum into $ctb and $ntb + + $i = $i + 1 + $tstart = 1640966400000 +endw + +$totalTblNum = $tbNum * 2 +sql show tables +if $rows != $totalTblNum then + return -1 +endi + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +$totalRowsOfStb = $rowNum * $tbNum +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + $dnodeId = $data[0][3] +elif $data[0][6] == LEADER then + $dnodeId = $data[0][5] +endi +elif $data[0][8] == LEADER then + $dnodeId = $data[0][7] +else + print ====> no leader vnode!!! + return -1 +endi + +$dnodeId = dnode . $dnodeId +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] +elif $data[0][6] == LEADER then + print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] +endi +elif $data[0][8] == LEADER then + print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] +else + print ====> no leader vnode!!! + return -1 +endi + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> stop and start all dnode(not include the dnode where mnode is located), then query +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode2 -s start + +$loop_cnt = 0 +check_vg_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> after restart dnode, vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != FLLOWER then + goto check_vg_ready_1 + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][6] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready_1 + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][8] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready_1 + endi + if $data[0][6] != FLLOWER then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +vg_ready_1: + +print ====> after restart dnode2/dnode3/dnode4, query stb/ntb count(*) +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> once stop one dnode by loop, and do query every time +$i = 2 +loop_stop_dnode: +$dnodeId = dnode . $i +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + +check_vg_ready_3: +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + if $data[0][6] == LEADER then + goto check_vg_ready_3 + endi + if $data[0][8] == LEADER then + goto check_vg_ready_3 + endi + print ---- vgroup $data[0][0] leader locating dnode $data[0][5] +elif $data[0][6] == LEADER then + if $data[0][4] == LEADER then + goto check_vg_ready_3 + endi + if $data[0][8] == LEADER then + goto check_vg_ready_3 + endi + print ---- vgroup $data[0][0] leader locating dnode $data[0][7] +endi +elif $data[0][8] == LEADER then + if $data[0][4] == LEADER then + goto check_vg_ready_3 + endi + if $data[0][6] == LEADER then + goto check_vg_ready_3 + endi + print ---- vgroup $data[0][0] leader locating dnode $data[0][9] +else + print ====> no leader vnode!!! + return -1 +endi + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $rowNum then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +$i = $i + 1 +if $i <= 4 then + print ====> start $dnodeId + system sh/exec.sh -n $dnodeId -s start + goto loop_stop_dnode +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim new file mode 100644 index 0000000000..d934e82b98 --- /dev/null +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -0,0 +1,469 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][0] != 1 then + return -1 +endi +if $data[0][4] != ready then + goto check_dnode_ready +endi + +#sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +check_dnode_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> dnodes not ready! + return -1 +endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][4] != ready then + goto check_dnode_ready_1 +endi +if $data[1][4] != ready then + goto check_dnode_ready_1 +endi +if $data[2][4] != ready then + goto check_dnode_ready_1 +endi +if $data[3][4] != ready then + goto check_dnode_ready_1 +endi + +$vgroups = 1 +$replica = 3 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $rows != 3 then + return -1 +endi +if $data(db)[20] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != FLLOWER then + goto check_vg_ready + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + goto vg_ready +endi +if $data[0][6] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + goto vg_ready +endi +if $data[0][8] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready + endi + if $data[0][6] != FLLOWER then + goto check_vg_ready + endi + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + goto vg_ready +endi +vg_ready: + +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +#$rowNum = 10 +#$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + +# $x = 0 +# while $x < $rowNum +# $binary = ' . binary +# $binary = $binary . $i +# $binary = $binary . ' +# +# sql insert into $ctb values ($tstart , $i , $x , $binary ) +# sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) +# $tstart = $tstart + 1 +# $x = $x + 1 +# endw + +# print ====> insert rows: $rowNum into $ctb and $ntb + + $i = $i + 1 +# $tstart = 1640966400000 +endw + +$totalTblNum = $tbNum * 2 +sql show tables +if $rows != $totalTblNum then + return -1 +endi + +print ====> create a normal table for interaction between main and back threads +sql create table interaction (ts timestamp, flag binary(10), childrows int, stbrows int) + +print ====> start to run_back to insert data +run_back tsim/tmq/insertDataByRunBack.sim + + +print ====> waiting insert thread starting insert data +waiting_insert_data: +sql select * from interaction +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] +if $rows == 1 then + if $data[0][1] == working then + goto start_switch_leader + endi +endi +goto waiting_insert_data + +start_switch_leader: + +$switch_loop_cnt = 0 +switch_leader_loop: + +print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + $dnodeId = $data[0][3] +elif $data[0][6] == LEADER then + $dnodeId = $data[0][5] +endi +elif $data[0][8] == LEADER then + $dnodeId = $data[0][7] +else + print ====> no leader vnode!!! + return -1 +endi + +$dnodeId = dnode . $dnodeId +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + +$loop_cnt = 0 +check_vg_ready_2: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> vgroups switch fail!!! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != FLLOWER then + goto check_vg_ready_2 + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] + goto vg_ready_2 +endi +if $data[0][6] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready_2 + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] + goto vg_ready_2 +endi +if $data[0][8] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready_2 + endi + if $data[0][6] != FLLOWER then + goto check_vg_ready_2 + endi + print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] + goto vg_ready_2 +endi +vg_ready_2: + +$switch_loop_cnt = $switch_loop_cnt + 1 +if $switch_loop_cnt < 3 then + print ====> start $dnodeId + system sh/exec.sh -n $dnodeId -s start + goto switch_leader_loop +endi + +loop_switch_end: + +print ====> notify insert thread to stop insert data +sql insert into interaction values (now, 'stop', 0, 0); + +print ====> waiting insert thread to stop insert data +waiting_stop_data: +sql select * from interaction +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $rows == 3 then + if $data[2][1] == end then + $totalRowsOfCtb = $data[2][2] + $totalRowsOfStb = $data[2][3] + goto check_affected_rows + endi +endi +goto waiting_stop_data + +check_affected_rows: + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> stop and start all dnode(not include the dnode where mnode is located), then query +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode2 -s start + +$loop_cnt = 0 +check_vg_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> after restart dnode, vgroups not ready! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[10][6] $data[0][11] $data[0][12] $data[0][13] +if $rows != $vgroups then + return -1 +endi +if $data[0][4] == LEADER then + if $data[0][6] != FLLOWER then + goto check_vg_ready_1 + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][6] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready_1 + endi + if $data[0][8] != FLLOWER then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +if $data[0][8] == LEADER then + if $data[0][4] != FLLOWER then + goto check_vg_ready_1 + endi + if $data[0][6] != FLLOWER then + goto check_vg_ready_1 + endi + goto vg_ready_1 +endi +vg_ready_1: + +print ====> after restart dnode2/dnode3/dnode4, query stb/ntb count(*) +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +print ====> once stop one dnode by loop, and do query every time +$i = 2 +loop_stop_dnode: +$dnodeId = dnode . $i +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + +check_vg_ready_3: +sql show vgroups +print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] +if $data[0][4] == LEADER then + if $data[0][6] == LEADER then + goto check_vg_ready_3 + endi + if $data[0][8] == LEADER then + goto check_vg_ready_3 + endi + print ---- vgroup $data[0][0] leader locating dnode $data[0][5] +elif $data[0][6] == LEADER then + if $data[0][4] == LEADER then + goto check_vg_ready_3 + endi + if $data[0][8] == LEADER then + goto check_vg_ready_3 + endi + print ---- vgroup $data[0][0] leader locating dnode $data[0][7] +endi +elif $data[0][8] == LEADER then + if $data[0][4] == LEADER then + goto check_vg_ready_3 + endi + if $data[0][6] == LEADER then + goto check_vg_ready_3 + endi + print ---- vgroup $data[0][0] leader locating dnode $data[0][9] +else + print ====> no leader vnode!!! + return -1 +endi + +sql select count(*) from ntb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from ctb0 +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfCtb then + return -1 +endi + +sql select count(*) from stb +print rows: $rows +print $data[0][0] $data[0][1] +if $data[0][0] != $totalRowsOfStb then + return -1 +endi + +$i = $i + 1 +if $i <= 4 then + print ====> start $dnodeId + system sh/exec.sh -n $dnodeId -s start + goto loop_stop_dnode +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT From 1c94bbb5e4bfe3c714d874f4092a0f7347f9c59c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Apr 2022 16:11:16 +0800 Subject: [PATCH 31/31] enh(query): set the status of stream scan operator. --- source/common/src/tmsg.c | 4 ++++ source/libs/executor/src/executor.c | 2 ++ source/libs/executor/src/executorimpl.c | 4 ++++ source/libs/executor/src/scanoperator.c | 11 +++++++++-- source/libs/function/src/builtinsimpl.c | 4 ---- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5c97743a62..0352999b3b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -52,6 +52,10 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { if (pIter->len == 0) { pIter->len += sizeof(SSubmitReq); } else { + if (pIter->len >= pIter->totalLen) { + ASSERT(0); + } + SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); ASSERT(pIter->len > 0); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4af7e563e6..4863b03fb9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -34,6 +34,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); } else { + pOperator->status = OP_NOT_OPENED; + SStreamBlockScanInfo* pInfo = pOperator->info; // the block type can not be changed in the streamscan operators diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 366da58079..87ecdde3ab 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4931,6 +4931,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) SSDataBlock* pRes = pInfo->pRes; blockDataCleanup(pRes); + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } #if 0 if (pProjectInfo->existDataBlock) { // TODO refactor diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b3050006b7..6b06f3e89b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -539,7 +539,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) SStreamBlockScanInfo* pInfo = pOperator->info; pTaskInfo->code = pOperator->_openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -547,6 +547,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) size_t total = taosArrayGetSize(pInfo->pBlockLists); if (pInfo->validBlockIndex >= total) { doClearBufferedBlocks(pInfo); + pOperator->status = OP_EXEC_DONE; return NULL; } @@ -560,11 +561,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { terrno = pTaskInfo->code; + pOperator->status = OP_EXEC_DONE; return NULL; } if (pBlockInfo->rows == 0) { - return NULL; + break; } SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle); @@ -583,6 +585,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) if (pInfo->pRes->pDataBlock == NULL) { // TODO add log + pOperator->status = OP_EXEC_DONE; pTaskInfo->code = terrno; return NULL; } @@ -594,6 +597,10 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) pInfo->numOfExec++; pInfo->numOfRows += pBlockInfo->rows; + if (pBlockInfo->rows == 0) { + pOperator->status = OP_EXEC_DONE; + } + return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; } } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5a32a6ffec..2b1e4b9406 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -866,10 +866,6 @@ int32_t firstFunction(SqlFunctionCtx *pCtx) { } int32_t lastFunction(SqlFunctionCtx *pCtx) { - if (pCtx->order != TSDB_ORDER_DESC) { - return 0; - } - int32_t numOfElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);