From de55d16b60f1782a655b44b66cf4cbf878dc8d55 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 26 Aug 2022 10:59:48 +0800 Subject: [PATCH 1/4] enh: downgrade value type to speed up filter --- source/libs/scalar/inc/sclInt.h | 2 + source/libs/scalar/src/scalar.c | 85 +++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index d423b92da7..36d2c5a49c 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -45,6 +45,8 @@ typedef struct SScalarCtx { #define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) //#define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type) && (((SValueNode *)_node)->placeholderNo <= 0)) #define SCL_IS_NULL_VALUE_NODE(_node) ((QUERY_NODE_VALUE == nodeType(_node)) && (TSDB_DATA_TYPE_NULL == ((SValueNode *)_node)->node.resType.type)) +#define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN) +#define SCL_DOWNGRADE_DATETYPE(_type) ((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT) #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 6634a29f40..cd1f6624bd 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -9,6 +9,7 @@ #include "scalar.h" #include "tudf.h" #include "ttime.h" +#include "tcompare.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type @@ -219,6 +220,82 @@ void sclFreeParamList(SScalarParam *param, int32_t paramNum) { taosMemoryFree(param); } +void sclDowngradeValueType(SValueNode *valueNode) { + switch (valueNode->node.resType.type) { + case TSDB_DATA_TYPE_BIGINT: { + int8_t i8 = valueNode->datum.i; + if (i8 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_TINYINT; + *(int8_t*)&valueNode->typeData = i8; + break; + } + int16_t i16 = valueNode->datum.i; + if (i16 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_SMALLINT; + *(int16_t*)&valueNode->typeData = i16; + break; + } + int32_t i32 = valueNode->datum.i; + if (i32 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_INT; + *(int32_t*)&valueNode->typeData = i32; + break; + } + break; + } + case TSDB_DATA_TYPE_UBIGINT:{ + uint8_t u8 = valueNode->datum.i; + if (u8 == valueNode->datum.i) { + int8_t i8 = valueNode->datum.i; + if (i8 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_TINYINT; + *(int8_t*)&valueNode->typeData = i8; + } else { + valueNode->node.resType.type = TSDB_DATA_TYPE_UTINYINT; + *(uint8_t*)&valueNode->typeData = u8; + } + break; + } + uint16_t u16 = valueNode->datum.i; + if (u16 == valueNode->datum.i) { + int16_t i16 = valueNode->datum.i; + if (i16 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_SMALLINT; + *(int16_t*)&valueNode->typeData = i16; + } else { + valueNode->node.resType.type = TSDB_DATA_TYPE_USMALLINT; + *(uint16_t*)&valueNode->typeData = u16; + } + break; + } + uint32_t u32 = valueNode->datum.i; + if (u32 == valueNode->datum.i) { + int32_t i32 = valueNode->datum.i; + if (i32 == valueNode->datum.i) { + valueNode->node.resType.type = TSDB_DATA_TYPE_INT; + *(int32_t*)&valueNode->typeData = i32; + } else { + valueNode->node.resType.type = TSDB_DATA_TYPE_UINT; + *(uint32_t*)&valueNode->typeData = u32; + } + break; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + float f = valueNode->datum.d; + if (FLT_EQUAL(f, valueNode->datum.d)) { + valueNode->node.resType.type = TSDB_DATA_TYPE_FLOAT; + *(float*)&valueNode->typeData = f; + break; + } + break; + } + default: + break; + } +} + int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_LEFT_VALUE: { @@ -675,6 +752,10 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } } + + if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { + sclDowngradeValueType(valueNode); + } } if (node->pRight && (QUERY_NODE_VALUE == nodeType(node->pRight))) { @@ -692,6 +773,10 @@ EDealRes sclRewriteNonConstOperator(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } } + + if (SCL_IS_COMPARISON_OPERATOR(node->opType) && SCL_DOWNGRADE_DATETYPE(valueNode->node.resType.type)) { + sclDowngradeValueType(valueNode); + } } if (node->pRight && (QUERY_NODE_NODE_LIST == nodeType(node->pRight))) { From 083ce4e67dec147a4c4df0b43acb5785ea386923 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 26 Aug 2022 15:43:20 +0800 Subject: [PATCH 2/4] fix: use sizeof(SBlockL) from aBlockL array's element size --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index b9f3897674..61c6877555 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -476,7 +476,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (code) goto _err; if (!state->aBlockL) { - state->aBlockL = taosArrayInit(0, sizeof(SBlockIdx)); + state->aBlockL = taosArrayInit(0, sizeof(SBlockL)); } else { taosArrayClear(state->aBlockL); } From ab85eeaaaa438d37008a83d360306032e6f8dba5 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 26 Aug 2022 16:15:08 +0800 Subject: [PATCH 3/4] fix(query): restrict using JSON type as input in string functions TD-18664 --- source/libs/function/src/builtins.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index ed82e4cb50..864424846e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -303,7 +303,7 @@ static int32_t translateInOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t le } SExprNode* pPara1 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - if (!IS_VAR_DATA_TYPE(pPara1->resType.type)) { + if (!IS_STR_DATA_TYPE(pPara1->resType.type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -317,7 +317,7 @@ static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len } SExprNode* pPara1 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); - if (!IS_VAR_DATA_TYPE(pPara1->resType.type)) { + if (!IS_STR_DATA_TYPE(pPara1->resType.type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -546,7 +546,7 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t // param2 if (3 == numOfParams) { uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type; - if (!IS_VAR_DATA_TYPE(para3Type)) { + if (!IS_STR_DATA_TYPE(para3Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -593,7 +593,7 @@ static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int // param2 if (3 == numOfParams) { uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type; - if (!IS_VAR_DATA_TYPE(para3Type)) { + if (!IS_STR_DATA_TYPE(para3Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1388,7 +1388,7 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } // set result type - if (IS_VAR_DATA_TYPE(colType)) { + if (IS_STR_DATA_TYPE(colType)) { pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; } else { pFunc->node.resType = (SDataType){.bytes = tDataTypes[colType].bytes, .type = colType}; @@ -1431,7 +1431,7 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } // set result type - if (IS_VAR_DATA_TYPE(colType)) { + if (IS_STR_DATA_TYPE(colType)) { pFunc->node.resType = (SDataType){.bytes = pCol->resType.bytes, .type = colType}; } else { pFunc->node.resType = (SDataType){.bytes = tDataTypes[colType].bytes, .type = colType}; @@ -1514,7 +1514,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) for (int32_t i = 1; i < 3; ++i) { nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, i)); paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; - if (!IS_VAR_DATA_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) { + if (!IS_STR_DATA_TYPE(paraType) || QUERY_NODE_VALUE != nodeType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1682,7 +1682,8 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - if (!IS_VAR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_STR_DATA_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1714,7 +1715,7 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t for (int32_t i = 0; i < numOfParams; ++i) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, i); uint8_t paraType = ((SExprNode*)pPara)->resType.type; - if (!IS_VAR_DATA_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { + if (!IS_STR_DATA_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } if (TSDB_DATA_TYPE_NCHAR == paraType) { @@ -1770,7 +1771,7 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) uint8_t para0Type = pPara0->resType.type; uint8_t para1Type = pPara1->resType.type; - if (!IS_VAR_DATA_TYPE(para0Type) || !IS_INTEGER_TYPE(para1Type)) { + if (!IS_STR_DATA_TYPE(para0Type) || !IS_INTEGER_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1802,7 +1803,7 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { uint8_t para2Type = pFunc->node.resType.type; int32_t para2Bytes = pFunc->node.resType.bytes; - if (IS_VAR_DATA_TYPE(para2Type)) { + if (IS_STR_DATA_TYPE(para2Type)) { para2Bytes -= VARSTR_HEADER_SIZE; } if (para2Bytes <= 0 || para2Bytes > 4096) { // cast dst var type length limits to 4096 bytes @@ -1859,7 +1860,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - if (!IS_VAR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { + if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1878,7 +1879,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; - if ((!IS_VAR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || + if ((!IS_STR_DATA_TYPE(para1Type) && !IS_INTEGER_TYPE(para1Type) && TSDB_DATA_TYPE_TIMESTAMP != para1Type) || !IS_INTEGER_TYPE(para2Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1911,7 +1912,7 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le for (int32_t i = 0; i < 2; ++i) { uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; - if (!IS_VAR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { + if (!IS_STR_DATA_TYPE(paraType) && !IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } } From 0fbb4d7f9ad337fa4231a1d5cd47e6013a9bf7a9 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 26 Aug 2022 16:15:08 +0800 Subject: [PATCH 4/4] fix(query): restrict using JSON type as input in string functions TD-18664 --- source/libs/function/src/builtins.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 864424846e..b7cd02befd 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1682,8 +1682,7 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_STR_DATA_TYPE(paraType)) { + if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); }