From ef0d9262c98a85048398ca648a8e3b0cb9511fa4 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 14 Feb 2022 19:38:06 +0800 Subject: [PATCH 1/9] feature/qnode --- include/nodes/nodes.h | 1 + source/libs/function/inc/tscalar.h | 41 ++++++++++++ source/libs/function/src/taggfunction.c | 2 +- source/libs/function/src/tbinoperator.c | 12 ++-- source/libs/function/src/tscalar.c | 73 ++++++++++++++++++++++ source/libs/function/src/tscalarfunction.c | 3 + source/nodes/src/nodesUtilFuncs.c | 32 +++++++++- 7 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 source/libs/function/inc/tscalar.h create mode 100644 source/libs/function/src/tscalar.c diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index b182063808..e692b58e54 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -337,6 +337,7 @@ SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SNode* nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); +void *nodesGetValueFromNode(SValueNode *pNode); typedef enum EDealRes { DEAL_RES_CONTINUE = 1, diff --git a/source/libs/function/inc/tscalar.h b/source/libs/function/inc/tscalar.h new file mode 100644 index 0000000000..af0ca2dc4c --- /dev/null +++ b/source/libs/function/inc/tscalar.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_SCALAR_H +#define TDENGINE_SCALAR_H + +#ifdef __cplusplus +extern "C" { +#endif + + +#define sclFatal(...) qFatal(__VA_ARGS__) +#define sclError(...) qError(__VA_ARGS__) +#define sclWarn(...) qWarn(__VA_ARGS__) +#define sclInfo(...) qInfo(__VA_ARGS__) +#define sclDebug(...) qDebug(__VA_ARGS__) +#define sclTrace(...) qTrace(__VA_ARGS__) + +#define SCL_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define SCL_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SCALAR_H diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index d0d89611c4..881845f42b 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3243,7 +3243,7 @@ static void arithmetic_function(SqlFunctionCtx *pCtx) { SScalarFuncParam output = {0}; output.data = pCtx->pOutput; - evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); + //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c index 4de11cbe71..72144bf130 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/function/src/tbinoperator.c @@ -465,18 +465,16 @@ void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { switch (binFunctionId) { - case TSDB_BINARY_OP_ADD: + case OP_TYPE_ADD: return vectorAdd; - case TSDB_BINARY_OP_SUBTRACT: + case OP_TYPE_SUB: return vectorSub; - case TSDB_BINARY_OP_MULTIPLY: + case OP_TYPE_MULTI: return vectorMultiply; - case TSDB_BINARY_OP_DIVIDE: + case OP_TYPE_DIV: return vectorDivide; - case TSDB_BINARY_OP_REMAINDER: + case OP_TYPE_MOD: return vectorRemainder; - case TSDB_BINARY_OP_CONCAT: - return vectorConcat; default: assert(0); return NULL; diff --git a/source/libs/function/src/tscalar.c b/source/libs/function/src/tscalar.c new file mode 100644 index 0000000000..6a4a8975af --- /dev/null +++ b/source/libs/function/src/tscalar.c @@ -0,0 +1,73 @@ +#include "nodes.h" +#include "tscalar.h" + +EDealRes sclCalculateConstants(SNode** pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_OPERATOR != nodeType(*pNode)) { + sclError("invalid node type for calculating constants, type:%d", ); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SOperatorNode *oper = (SOperatorNode *)*pNode; + if (NULL == oper->pLeft || NULL == oper->pRight) { + sclError("invalid operation node, left:%p, right:%p", oper->pLeft, oper->pRight); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + if (QUERY_NODE_VALUE != nodeType(oper->pLeft) || QUERY_NODE_VALUE != nodeType(oper->pRight)) { + sclError("invalid operation node, leftType:%d, rightType:%d", nodeType(oper->pLeft), nodeType(oper->pRight)); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = oper->node.resType; + + SValueNode *leftValue = (SValueNode *)oper->pLeft; + SValueNode *rightValue = (SValueNode *)oper->pRight; + + SScalarFuncParam leftParam = {0}, rightParam = {0}; + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(oper->opType); + setScalarFuncParam(&leftParam, leftValue->node.resType, 0, nodesGetValueFromNode(leftValue), 1); + setScalarFuncParam(&rightParam, rightValue->node.resType, 0, nodesGetValueFromNode(rightValue), 1); + + OperatorFn(&leftParam, &rightParam, nodesGetValueFromNode(res), TSDB_ORDER_ASC); + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + return DEAL_RES_CONTINUE; +} + +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { + if (NULL == pNode) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + + nodesRewriteNodePostOrder(&pNode, sclCalculateConstants, (void *)&code); + + if (code) { + nodesDestroyNode(pNode); + SCL_ERR_RET(code); + } + + *pRes = pNode; + + SCL_RET(code); +} + + + diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 90d25b3e4a..50aaa07757 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -273,6 +273,7 @@ bool isStringOp(int32_t op) { return op == TSDB_BINARY_OP_CONCAT; } +#if 0 int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)) { if (pExprs == NULL) { @@ -361,6 +362,8 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa return 0; } +#endif + SScalarFunctionInfo scalarFunc[8] = { {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index e0e589157c..e13cf025cc 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -152,6 +152,36 @@ void nodesDestroyList(SNodeList* pList) { tfree(pList); } +void *nodesGetValueFromNode(SValueNode *pNode) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_BOOL: + return (void *)&pNode->datum.b; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return (void *)&pNode->datum.i; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + return (void *)&pNode->datum.u; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + return (void *)&pNode->datum.d; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + return (void *)pNode->datum.p; + default: + break; + } + + return NULL; +} + bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || QUERY_NODE_FUNCTION == type); @@ -209,4 +239,4 @@ bool nodesIsTimeorderQuery(const SNode* pQuery) { bool nodesIsTimelineQuery(const SNode* pQuery) { return false; -} \ No newline at end of file +} From 75ede63c8a421b0d263b55aee2ca282cc73a45ec Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 17 Feb 2022 09:34:55 +0800 Subject: [PATCH 2/9] feature/qnode --- include/common/tcompare.h | 10 +- include/common/ttypes.h | 15 +- include/libs/function/functionMgt.h | 10 +- include/nodes/querynodes.h | 6 +- include/util/compare.h | 18 +- include/util/tdef.h | 18 +- include/util/types.h | 1 + source/common/src/tcompare.c | 61 ++- source/libs/executor/src/tfilter.c | 37 +- source/libs/function/inc/tscalar.h | 3 + source/libs/function/src/tbinoperator.c | 676 ++++++++++++++++++++++++ source/libs/function/src/tscalar.c | 272 +++++++++- source/util/src/compare.c | 38 +- 13 files changed, 1110 insertions(+), 55 deletions(-) diff --git a/include/common/tcompare.h b/include/common/tcompare.h index 8476a79e92..5dcbf37b90 100644 --- a/include/common/tcompare.h +++ b/include/common/tcompare.h @@ -23,8 +23,12 @@ extern "C" { #endif -int32_t compareStrPatternComp(const void* pLeft, const void* pRight); -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight); +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight); +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight); + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight); +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight); + __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); @@ -33,4 +37,4 @@ int32_t doCompare(const char* a, const char* b, int32_t type, size_t size) } #endif -#endif /*_TD_TCOMPARE_H_*/ \ No newline at end of file +#endif /*_TD_TCOMPARE_H_*/ diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 505685f8d5..afa605044a 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -78,9 +78,12 @@ typedef struct { case TSDB_DATA_TYPE_UINT: \ (_v) = (_finalType)GET_UINT32_VAL(_data); \ break; \ - default: \ + case TSDB_DATA_TYPE_INT: \ (_v) = (_finalType)GET_INT32_VAL(_data); \ break; \ + default: \ + (_v) = (_finalType)varDataLen(_data); \ + break; \ } \ } while (0) @@ -115,9 +118,13 @@ typedef struct { case TSDB_DATA_TYPE_UINT: \ *(uint32_t *)(_v) = (uint32_t)(_data); \ break; \ - default: \ + case TSDB_DATA_TYPE_INT: \ *(int32_t *)(_v) = (int32_t)(_data); \ break; \ + default: \ + (void *)(_v) = (void *)(_data); \ + (void *)(_data) = NULL; \ + break; \ } \ } while (0) @@ -138,6 +145,9 @@ typedef struct { #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) +#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) +#define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) + static FORCE_INLINE bool isNull(const void *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -205,6 +215,7 @@ void* getDataMax(int32_t type); #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) +#define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL) #ifdef __cplusplus } diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index c3c8b5c4ce..bdf9a925a8 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -118,6 +118,13 @@ typedef struct SFuncExecFuncs { FExecFinalize finalize; } SFuncExecFuncs; +typedef int32_t (*FScalarExecProcess)(SScalarFuncParam *pInput, int32_t inputNum, SScalarFuncParam *pOutput); + +typedef struct SScalarFuncExecFuncs { + FScalarExecProcess process; +} SScalarFuncExecFuncs; + + int32_t fmFuncMgtInit(); int32_t fmGetHandle(FuncMgtHandle* pHandle); @@ -136,7 +143,8 @@ bool fmIsTimeorderFunc(int32_t funcId); int32_t fmFuncScanType(int32_t funcId); -int32_t fmGetFuncExecFuncs(FuncMgtHandle handle, int32_t funcId, SFuncExecFuncs* pFpSet); +int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet); +int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet); #ifdef __cplusplus } diff --git a/include/nodes/querynodes.h b/include/nodes/querynodes.h index 646ea63c83..f41a5faec2 100644 --- a/include/nodes/querynodes.h +++ b/include/nodes/querynodes.h @@ -93,6 +93,10 @@ typedef enum EOperatorType { OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH, + OP_TYPE_ISNULL, + OP_TYPE_NOTNULL, + OP_TYPE_BIT_AND, + OP_TYPE_BIT_OR, // json operator OP_TYPE_JSON_GET_VALUE, @@ -277,4 +281,4 @@ bool nodesIsTimelineQuery(const SNode* pQuery); } #endif -#endif /*_TD_QUERY_NODES_H_*/ \ No newline at end of file +#endif /*_TD_QUERY_NODES_H_*/ diff --git a/include/util/compare.h b/include/util/compare.h index 70a8134b35..0ae85418c7 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -49,10 +49,19 @@ int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, int32_t taosArrayCompareString(const void *a, const void *b); -int32_t setCompareBytes1(const void *pLeft, const void *pRight); -int32_t setCompareBytes2(const void *pLeft, const void *pRight); -int32_t setCompareBytes4(const void *pLeft, const void *pRight); -int32_t setCompareBytes8(const void *pLeft, const void *pRight); +int32_t setChkInBytes1(const void *pLeft, const void *pRight); +int32_t setChkInBytes2(const void *pLeft, const void *pRight); +int32_t setChkInBytes4(const void *pLeft, const void *pRight); +int32_t setChkInBytes8(const void *pLeft, const void *pRight); + +int32_t setChkNotInBytes1(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes2(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes4(const void *pLeft, const void *pRight); +int32_t setChkNotInBytes8(const void *pLeft, const void *pRight); + +int32_t compareChkInString(const void *pLeft, const void *pRight); +int32_t compareChkNotInString(const void *pLeft, const void *pRight); + int32_t compareInt8Val(const void *pLeft, const void *pRight); int32_t compareInt16Val(const void *pLeft, const void *pRight); @@ -74,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight); int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); -int32_t compareFindItemInSet(const void *pLeft, const void *pRight); int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); diff --git a/include/util/tdef.h b/include/util/tdef.h index 747020a4f9..292a3126c8 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -118,16 +118,18 @@ do { \ #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 -#define TSDB_RELATION_ISNULL 8 -#define TSDB_RELATION_NOTNULL 9 -#define TSDB_RELATION_IN 10 +#define TSDB_RELATION_NOT_LIKE 8 +#define TSDB_RELATION_ISNULL 9 +#define TSDB_RELATION_NOTNULL 10 +#define TSDB_RELATION_IN 11 +#define TSDB_RELATION_NOT_IN 12 -#define TSDB_RELATION_AND 11 -#define TSDB_RELATION_OR 12 -#define TSDB_RELATION_NOT 13 +#define TSDB_RELATION_AND 13 +#define TSDB_RELATION_OR 14 +#define TSDB_RELATION_NOT 15 -#define TSDB_RELATION_MATCH 14 -#define TSDB_RELATION_NMATCH 15 +#define TSDB_RELATION_MATCH 16 +#define TSDB_RELATION_NMATCH 17 #define TSDB_BINARY_OP_ADD 4000 #define TSDB_BINARY_OP_SUBTRACT 4001 diff --git a/include/util/types.h b/include/util/types.h index cb25448cc4..25688df56c 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -66,6 +66,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { // #else #define GET_FLOAT_VAL(x) (*(float *)(x)) #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_BIGINT_VAL(x, y) { (*(int64_t *)(x)) = (int64_t)(y); } #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c index 3c0e6df735..0ed490bed0 100644 --- a/source/common/src/tcompare.c +++ b/source/common/src/tcompare.c @@ -15,7 +15,7 @@ #include "tcompare.h" -int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); @@ -33,7 +33,11 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); @@ -47,6 +51,10 @@ int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; @@ -55,19 +63,42 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - return setCompareBytes1; + return setChkInBytes1; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - return setCompareBytes2; + return setChkInBytes2; case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_FLOAT: - return setCompareBytes4; + return setChkInBytes4; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: - return setCompareBytes8; + return setChkInBytes8; + default: + assert(0); + } + } + + if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkNotInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkNotInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkNotInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkNotInBytes8; default: assert(0); } @@ -88,9 +119,13 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } else if (optr == TSDB_RELATION_NMATCH) { comparFn = compareStrRegexCompNMatch; } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternComp; + comparFn = compareStrPatternMatch; + } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternNotMatch; } else if (optr == TSDB_RELATION_IN) { - comparFn = compareFindItemInSet; + comparFn = compareChkInString; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = compareChkNotInString; } else { /* normal relational comparFn */ comparFn = compareLenPrefixedStr; } @@ -104,9 +139,13 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } else if (optr == TSDB_RELATION_NMATCH) { comparFn = compareStrRegexCompNMatch; } else if (optr == TSDB_RELATION_LIKE) { - comparFn = compareWStrPatternComp; + comparFn = compareWStrPatternMatch; + } else if (optr == TSDB_RELATION_NOT_LIKE) { + comparFn = compareWStrPatternNotMatch; } else if (optr == TSDB_RELATION_IN) { - comparFn = compareFindItemInSet; + comparFn = compareChkInString; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = compareChkNotInString; } else { comparFn = compareLenPrefixedWStr; } @@ -218,4 +257,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { } } } -} \ No newline at end of file +} diff --git a/source/libs/executor/src/tfilter.c b/source/libs/executor/src/tfilter.c index 97dccb5c7b..c42260b214 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/executor/src/tfilter.c @@ -156,9 +156,11 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { } __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, - compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, + compareDoubleVal, compareLenPrefixedStr, compareStrPatternMatch, compareChkInString, compareWStrPatternMatch, compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, - setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch + setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, + compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { @@ -186,6 +188,29 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { assert(0); } } + + if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return 21; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return 22; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return 23; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return 24; + default: + assert(0); + } + } switch (type) { case TSDB_DATA_TYPE_BOOL: @@ -203,8 +228,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 20; } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ comparFn = 7; + } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = 26; } else if (optr == TSDB_RELATION_IN) { comparFn = 8; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = 25; } else { /* normal relational comparFn */ comparFn = 6; } @@ -219,8 +248,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 20; } else if (optr == TSDB_RELATION_LIKE) { comparFn = 9; + } else if (optr == TSDB_RELATION_LIKE) { + comparFn = 27; } else if (optr == TSDB_RELATION_IN) { comparFn = 8; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = 25; } else { comparFn = 10; } diff --git a/source/libs/function/inc/tscalar.h b/source/libs/function/inc/tscalar.h index af0ca2dc4c..da98069c86 100644 --- a/source/libs/function/inc/tscalar.h +++ b/source/libs/function/inc/tscalar.h @@ -19,6 +19,9 @@ extern "C" { #endif +typedef struct SScalarCalcContext { + +} SScalarCalcContext; #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c index 72144bf130..27b5d162a6 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/function/src/tbinoperator.c @@ -119,6 +119,70 @@ _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { } + +typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); + +int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { + return (int64_t)*((int8_t *)src + index); +} +int64_t getVectorBigintValue_UTINYINT(void *src, int32_t index) { + return (int64_t)*((uint8_t *)src + index); +} +int64_t getVectorBigintValue_SMALLINT(void *src, int32_t index) { + return (int64_t)*((int16_t *)src + index); +} +int64_t getVectorBigintValue_USMALLINT(void *src, int32_t index) { + return (int64_t)*((uint16_t *)src + index); +} +int64_t getVectorBigintValue_INT(void *src, int32_t index) { + return (int64_t)*((int32_t *)src + index); +} +int64_t getVectorBigintValue_UINT(void *src, int32_t index) { + return (int64_t)*((uint32_t *)src + index); +} +int64_t getVectorBigintValue_BIGINT(void *src, int32_t index) { + return (int64_t)*((int64_t *)src + index); +} +int64_t getVectorBigintValue_UBIGINT(void *src, int32_t index) { + return (int64_t)*((uint64_t *)src + index); +} +int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) { + return (int64_t)*((float *)src + index); +} +int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) { + return (int64_t)*((double *)src + index); +} +_getDoubleValue_fn_t getVectorBigintValueFn(int32_t srcType) { + _getDoubleValue_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorBigintValue_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorBigintValue_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorBigintValue_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorBigintValue_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorBigintValue_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorBigintValue_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorBigintValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorBigintValue_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorBigintValue_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorBigintValue_DOUBLE; + }else { + assert(0); + } + return p; +} + + + + typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index); void* getVectorValueAddr_TINYINT(void *src, int32_t index) { @@ -180,6 +244,337 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } + +int32_t vectorConvertImpl(SScalarFuncParam* pIn, SScalarFuncParam* pOut) { + int16_t inType = pIn->type; + int16_t inBytes = pIn->bytes; + char *input = pIn->data; + int16_t outType = pOut->type; + int16_t outBytes = pOut->bytes; + char *output = pOut->data; + + switch (outType) { + case TSDB_DATA_TYPE_BOOL: + if (inType == TSDB_DATA_TYPE_BINARY) { + for (int32_t i = 0; i < pIn->num; ++i) { + GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += sizeof(bool); + } + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + for (int32_t i = 0; i < pIn->num; ++i) { + GET_TYPED_DATA(*(bool *)output, bool, TSDB_DATA_TYPE_USMALLINT, &varDataLen(input)); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + qError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return; + } + + tmp[len] = 0; + int64_t value = strtoll(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + qError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return; + } + + tmp[len] = 0; + uint64_t value = strtoull(tmp, NULL, 10); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + uint64_t value = 0; + GET_TYPED_DATA(value, uint64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (inType == TSDB_DATA_TYPE_BINARY) { + int32_t bufSize = varDataLen(input) + 1; + char *tmp = malloc(bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input) >= bufSize) { + bufSize = varDataLen(input) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(input), varDataLen(input)); + tmp[varDataLen(input)] = 0; + + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else if (inType == TSDB_DATA_TYPE_NCHAR) { + int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + char *tmp = calloc(1, bufSize); + if (NULL == tmp) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); + if (len < 0){ + qError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return; + } + + tmp[len] = 0; + double value = strtod(tmp, NULL); + SET_TYPED_DATA(output, outType, value); + + input += varDataLen(input) + VARSTR_HEADER_SIZE; + output += tDataTypes[outType].bytes; + } + + tfree(tmp); + } else { + for (int32_t i = 0; i < pIn->num; ++i) { + int64_t value = 0; + GET_TYPED_DATA(value, int64_t, inType, input); + SET_TYPED_DATA(output, outType, value); + + input += tDataTypes[inType].bytes; + output += tDataTypes[outType].bytes; + } + } + break; + default: + qError("invalid convert output type:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { +/* NULL BOOL TINY SMAL INT BIG FLOA DOUB BINA TIME NCHA UTIN USMA UINT UBIG VARC VARB JSON DECI BLOB */ +/*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*BOOL*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 0, 12, 13, 14, 7, 7, 0, 0, 0, +/*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 7, 0, 0, 0, +/*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 7, 7, 0, 0, 0, +/*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 7, 7, 0, 0, 0, +/*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 0, 7, 5, 5, 5, 7, 7, 7, 0, 0, 0, +/*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 7, 7, 0, 0, 0, +/*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0, 0, 0, +/*BINA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, +/*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 9, 9, 9, 7, 7, 7, 0, 0, 0, +/*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, 0, +/*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 7, 7, 0, 0, 0, +/*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 7, 7, 0, 0, 0, +/*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 7, 7, 0, 0, 0, +/*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 0, 0, 0, +/*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +/*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +int32_t vectorConvert(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, SScalarFuncParam* pLeftOut, SScalarFuncParam* pRightOut) { + if (pLeft->type == pRight->type) { + return TSDB_CODE_SUCCESS; + } + + SScalarFuncParam *param1 = NULL, *paramOut1 = NULL; + SScalarFuncParam *param2 = NULL, *paramOut2 = NULL; + int32_t code = 0; + + if (pLeft->type < pRight->type) { + param1 = pLeft; + param2 = pRight; + paramOut1 = pLeftOut; + paramOut2 = pRightOut; + } else { + param1 = pRight; + param2 = pLeft; + paramOut1 = pRightOut; + paramOut2 = pLeftOut; + } + + int8_t type = gConvertTypes[param1->type][param2->type]; + if (0 == type) { + return TSDB_CODE_SUCCESS; + } + + if (type != param1->type) { + paramOut1->bytes = param1->bytes; + paramOut1->type = type; + paramOut1->num = param1->num; + paramOut1->data = malloc(paramOut1->num * tDataTypes[paramOut1->type].bytes); + if (NULL == paramOut1->data) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + code = vectorConvertImpl(param1, paramOut1); + if (code) { + tfree(paramOut1->data); + return code; + } + } + + if (type != param2->type) { + paramOut2->bytes = param2->bytes; + paramOut2->type = type; + paramOut2->num = param2->num; + paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); + if (NULL == paramOut2->data) { + tfree(paramOut1->data) + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + code = vectorConvertImpl(param2, paramOut2); + if (code) { + tfree(paramOut1->data) + tfree(paramOut2->data); + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -463,6 +858,255 @@ void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } + +void vectorBitAnd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + int64_t *output=(int64_t *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + + SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + } + } +} + +void vectorBitOr(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + int64_t *output=(int64_t *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); + _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + + SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_BIGINT_NULL(output); + continue; + } + SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + } + } +} + + +void vectorCompareImpl(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord, int32_t optr) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int8_t funcIdx = filterGetCompFuncIdx(pLeft->type, optr); + __compar_fn_t fp = gDataCompare[funcIdx]; + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i)); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } else if (pLeft->num == 1) { + void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i)); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } else if (pRight->num == 1) { + void *rightData = getVectorValueAddrFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData); + + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } + } +} + +void vectorCompare(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord, int32_t optr) { + SScalarFuncParam pLeftOut = {0}; + SScalarFuncParam pRightOut = {0}; + + vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); + + SScalarFuncParam *param1 = NULL; + SScalarFuncParam *param2 = NULL; + + int32_t type = 0; + if (pLeftOut->type) { + param1 = &pLeftOut; + } else { + param1 = pLeft; + } + + if (pRightOut->type) { + param2 = &pRightOut; + } else { + param2 = pRight; + } + + vectorCompareImpl(pLeftOut, pRightOut, out, _ord, TSDB_RELATION_GREATER); +} + +void vectorGreater(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER); +} + +void vectorGreaterEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL); +} + +void vectorLower(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS); +} + +void vectorLowerEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL); +} + +void vectorEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL); +} + +void vectorNotEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL); +} + +void vectorIn(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN); +} + +void vectorNotIn(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN); +} + +void vectorLike(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE); +} + +void vectorNotLike(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE); +} + +void vectorMatch(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH); +} + +void vectorNotMatch(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH); +} + +void vectorIsNull(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + res = true; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } +} + +void vectorNotNull(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + bool res = false; + + bool *output=(bool *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + res = false; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + continue; + } + + res = true; + SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + } +} + + _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { switch (binFunctionId) { case OP_TYPE_ADD: @@ -475,6 +1119,38 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorDivide; case OP_TYPE_MOD: return vectorRemainder; + case OP_TYPE_GREATER_THAN: + return vectorGreater; + case OP_TYPE_GREATER_EQUAL: + return vectorGreaterEqual; + case OP_TYPE_LOWER_THAN: + return vectorLower; + case OP_TYPE_LOWER_EQUAL: + return vectorLowerEqual; + case OP_TYPE_EQUAL: + return vectorEqual; + case OP_TYPE_NOT_EQUAL: + return vectorNotEqual; + case OP_TYPE_IN: + return vectorIn; + case OP_TYPE_NOT_IN: + return vectorNotIn; + case OP_TYPE_LIKE: + return vectorLike; + case OP_TYPE_NOT_LIKE: + return vectorNotLike; + case OP_TYPE_MATCH: + return vectorMatch; + case OP_TYPE_NMATCH: + return vectorNotMatch; + case OP_TYPE_ISNULL: + return vectorIsNull; + case OP_TYPE_NOTNULL: + return vectorNotNull; + case OP_TYPE_BIT_AND: + return vectorBitAnd; + case OP_TYPE_BIT_OR: + return vectorBitOr; default: assert(0); return NULL; diff --git a/source/libs/function/src/tscalar.c b/source/libs/function/src/tscalar.c index 6a4a8975af..e374ce2734 100644 --- a/source/libs/function/src/tscalar.c +++ b/source/libs/function/src/tscalar.c @@ -1,26 +1,164 @@ #include "nodes.h" #include "tscalar.h" -EDealRes sclCalculateConstants(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode)) { - return DEAL_RES_CONTINUE; +int32_t sclGetOperatorParamNum(EOperatorType type) { + if (OP_TYPE_ISNULL == type || OP_TYPE_NOTNULL == type) { + return 1; } - if (QUERY_NODE_OPERATOR != nodeType(*pNode)) { - sclError("invalid node type for calculating constants, type:%d", ); + return 2; +} + +int32_t sclPrepareFunctionParams(SScalarFuncParam **pParams, SNodeList* pParameterList) { + *pParams = calloc(pParameterList->length, sizeof(SScalarFuncParam)); + if (NULL == *pParams) { + sclError("calloc %d failed", pParameterList->length * sizeof(SScalarFuncParam)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + SListCell *cell = pParameterList->pHead; + for (int32_t i = 0; i < pParameterList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + tfree(*pParams); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + if (QUERY_NODE_VALUE != nodeType(cell->pNode)) { + sclError("invalid node type in cell, type:%d", nodeType(cell->pNode)); + tfree(*pParams); + return TSDB_CODE_QRY_APP_ERROR; + } + + SValueNode *valueNode = (SValueNode *)cell->pNode; + pParams[i].data = nodesGetValueFromNode(valueNode); + pParams[i].num = 1; + pParams[i].type = valueNode->node.resType.type; + pParams[i].bytes = valueNode->node.resType.bytes; + + cell = cell->pNext; + } + + return TSDB_CODE_SUCCESS; +} + +EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { + SFunctionNode *node = (SFunctionNode *)*pNode; + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; - } + } + + SScalarFuncExecFuncs ffpSet = {0}; + int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); + if (code) { + sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + *(int32_t *)pContext = code; + return DEAL_RES_ERROR; + } + + SScalarFuncParam *input = NULL; + if (sclPrepareFunctionParams(&input, node->pParameterList)) { + return DEAL_RES_ERROR; + } + + SScalarFuncParam output = {0}; + code = (*ffpSet.process)(input, node->pParameterList->length, &output); + if (code) { + sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + *(int32_t *)pContext = code; + return DEAL_RES_ERROR; + } + + SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + SET_TYPED_DATA(nodesGetValueFromNode(res), output.type, output.data); + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + tfree(output.data); + + return DEAL_RES_CONTINUE; +} + +EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + if (LOGIC_COND_TYPE_NOT == node->condType && node->pParameterList->length > 1) { + sclError("invalid NOT operation parameter number, paramNum:%d", node->pParameterList->length); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + bool value = false; + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + if (QUERY_NODE_VALUE != nodeType(cell->pNode)) { + sclError("invalid node type in cell, type:%d", nodeType(cell->pNode)); + return TSDB_CODE_QRY_APP_ERROR; + } + + SValueNode *valueNode = (SValueNode *)cell->pNode; + GET_TYPED_DATA(value, bool, valueNode->node.resType.type, nodesGetValueFromNode(valueNode)); + if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + break; + } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + break; + } else if (LOGIC_COND_TYPE_NOT == node->condType) { + value = !value; + } + + cell = cell->pNext; + } + + SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = node->node.resType; + + SET_TYPED_DATA(nodesGetValueFromNode(res), res->node.resType.type, value); + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + return DEAL_RES_CONTINUE; +} + +EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { SOperatorNode *oper = (SOperatorNode *)*pNode; - if (NULL == oper->pLeft || NULL == oper->pRight) { + int32_t paramNum = sclGetOperatorParamNum(oper->opType); + if (NULL == oper->pLeft || (paramNum == 2 && NULL == oper->pRight)) { sclError("invalid operation node, left:%p, right:%p", oper->pLeft, oper->pRight); *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } - - if (QUERY_NODE_VALUE != nodeType(oper->pLeft) || QUERY_NODE_VALUE != nodeType(oper->pRight)) { - sclError("invalid operation node, leftType:%d, rightType:%d", nodeType(oper->pLeft), nodeType(oper->pRight)); + + if (QUERY_NODE_VALUE != nodeType(oper->pLeft) || (paramNum == 2 && QUERY_NODE_VALUE != nodeType(oper->pRight))) { + sclError("invalid operation node, leftType:%d, rightType:%d", nodeType(oper->pLeft), oper->pRight ? nodeType(oper->pRight) : 0); *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } @@ -37,10 +175,12 @@ EDealRes sclCalculateConstants(SNode** pNode, void* pContext) { SValueNode *leftValue = (SValueNode *)oper->pLeft; SValueNode *rightValue = (SValueNode *)oper->pRight; - SScalarFuncParam leftParam = {0}, rightParam = {0}; + SScalarFuncParam leftParam = {0}, rightParam = {0}; _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(oper->opType); - setScalarFuncParam(&leftParam, leftValue->node.resType, 0, nodesGetValueFromNode(leftValue), 1); - setScalarFuncParam(&rightParam, rightValue->node.resType, 0, nodesGetValueFromNode(rightValue), 1); + setScalarFuncParam(&leftParam, leftValue->node.resType.type, 0, nodesGetValueFromNode(leftValue), 1); + if (2 == paramNum) { + setScalarFuncParam(&rightParam, rightValue->node.resType.type, 0, nodesGetValueFromNode(rightValue), 1); + } OperatorFn(&leftParam, &rightParam, nodesGetValueFromNode(res), TSDB_ORDER_ASC); @@ -50,6 +190,91 @@ EDealRes sclCalculateConstants(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + +EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + return sclRewriteFunction(pNode, pContext); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + return sclRewriteLogic(pNode, pContext); + } + + if (QUERY_NODE_OPERATOR != nodeType(*pNode)) { + sclError("invalid node type for calculating constants, type:%d", ); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + return sclRewriteOperator(pNode, pContext); +} + +EDealRes sclCalculate(SNode** pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + return sclCalculateFunction(pNode, pContext); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + return sclCalculateLogic(pNode, pContext); + } + + if (QUERY_NODE_OPERATOR != nodeType(*pNode)) { + sclError("invalid node type for calculating constants, type:%d", ); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SOperatorNode *oper = (SOperatorNode *)*pNode; + int32_t paramNum = sclGetOperatorParamNum(oper->opType); + if (NULL == oper->pLeft || (paramNum == 2 && NULL == oper->pRight)) { + sclError("invalid operation node, left:%p, right:%p", oper->pLeft, oper->pRight); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + if (QUERY_NODE_VALUE != nodeType(oper->pLeft) || (paramNum == 2 && QUERY_NODE_VALUE != nodeType(oper->pRight))) { + sclError("invalid operation node, leftType:%d, rightType:%d", nodeType(oper->pLeft), oper->pRight ? nodeType(oper->pRight) : 0); + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + sclError("make value node failed"); + *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + res->node.resType = oper->node.resType; + + SValueNode *leftValue = (SValueNode *)oper->pLeft; + SValueNode *rightValue = (SValueNode *)oper->pRight; + + SScalarFuncParam leftParam = {0}, rightParam = {0}; + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(oper->opType); + setScalarFuncParam(&leftParam, leftValue->node.resType.type, 0, nodesGetValueFromNode(leftValue), 1); + if (2 == paramNum) { + setScalarFuncParam(&rightParam, rightValue->node.resType.type, 0, nodesGetValueFromNode(rightValue), 1); + } + + OperatorFn(&leftParam, &rightParam, nodesGetValueFromNode(res), TSDB_ORDER_ASC); + + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + + return DEAL_RES_CONTINUE; +} + + + int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { if (NULL == pNode) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -57,7 +282,26 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { int32_t code = 0; - nodesRewriteNodePostOrder(&pNode, sclCalculateConstants, (void *)&code); + nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code); + + if (code) { + nodesDestroyNode(pNode); + SCL_ERR_RET(code); + } + + *pRes = pNode; + + SCL_RET(code); +} + +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SSDataBlock *pDst) { + if (NULL == pNode) { + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + + nodesRewriteNodePostOrder(&pNode, sclCalculate, (void *)&code); if (code) { nodesDestroyNode(pNode); diff --git a/source/util/src/compare.c b/source/util/src/compare.c index a1c1625c34..ab353fcbb3 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -24,22 +24,48 @@ #include "types.h" #include "ulog.h" -int32_t setCompareBytes1(const void *pLeft, const void *pRight) { +int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; } -int32_t setCompareBytes2(const void *pLeft, const void *pRight) { +int32_t setChkInBytes2(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; } -int32_t setCompareBytes4(const void *pLeft, const void *pRight) { +int32_t setChkInBytes4(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; } -int32_t setCompareBytes8(const void *pLeft, const void *pRight) { +int32_t setChkInBytes8(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; } +int32_t setChkNotInBytes1(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; +} + +int32_t setChkNotInBytes2(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0; +} + +int32_t setChkNotInBytes4(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0; +} + +int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) { + return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; +} + + +int32_t compareChkInString(const void *pLeft, const void* pRight) { + return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} + +int32_t compareChkNotInString(const void *pLeft, const void* pRight) { + return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; +} + + int32_t compareInt8Val(const void *pLeft, const void *pRight) { int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); if (left > right) return 1; @@ -392,7 +418,3 @@ int32_t taosArrayCompareString(const void* a, const void* b) { return compareLenPrefixedStr(x, y); } - -int32_t compareFindItemInSet(const void *pLeft, const void* pRight) { - return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; -} From 76d5298ba1e73075f3d5dcc077644caf3e60ecef Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 17 Feb 2022 16:08:59 +0800 Subject: [PATCH 3/9] feature/qnode --- include/libs/function/function.h | 4 +- include/libs/function/functionMgt.h | 2 +- include/libs/nodes/querynodes.h | 2 - source/libs/function/inc/tbinoperator.h | 2 +- source/libs/function/inc/tscalar.h | 8 +- source/libs/function/inc/tscalarfunction.h | 6 +- source/libs/function/inc/tunaryoperator.h | 2 +- source/libs/function/src/taggfunction.c | 2 +- source/libs/function/src/tbinoperator.c | 68 +-- source/libs/function/src/tscalar.c | 564 +++++++++++++++------ source/libs/function/src/tscalarfunction.c | 20 +- 11 files changed, 465 insertions(+), 215 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index aef5f7fec4..dfcac7ca3f 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -226,13 +226,13 @@ typedef struct SAggFunctionInfo { int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; -struct SScalarFuncParam; +struct SScalarParam; typedef struct SScalarFunctionInfo { char name[FUNCTIONS_NAME_MAX_LENGTH]; int8_t type; // scalar function or aggregation function uint32_t functionId; // index of scalar function - void (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput); + void (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput); } SScalarFunctionInfo; typedef struct SMultiFunctionsDesc { diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 5078b00c8c..84b1f8d5bf 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -117,7 +117,7 @@ typedef struct SFuncExecFuncs { FExecFinalize finalize; } SFuncExecFuncs; -typedef int32_t (*FScalarExecProcess)(SScalarFuncParam *pInput, int32_t inputNum, SScalarFuncParam *pOutput); +typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef struct SScalarFuncExecFuncs { FScalarExecProcess process; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index b6439acdbe..cea0aa32a8 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -104,8 +104,6 @@ typedef enum EOperatorType { OP_TYPE_NMATCH, OP_TYPE_IS_NULL, OP_TYPE_IS_NOT_NULL, - OP_TYPE_BIT_AND, - OP_TYPE_BIT_OR, // json operator OP_TYPE_JSON_GET_VALUE, diff --git a/source/libs/function/inc/tbinoperator.h b/source/libs/function/inc/tbinoperator.h index c1b6b0bc31..678d6169ab 100644 --- a/source/libs/function/inc/tbinoperator.h +++ b/source/libs/function/inc/tbinoperator.h @@ -22,7 +22,7 @@ extern "C" { #include "tscalarfunction.h" -typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order); +typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); bool isBinaryStringOp(int32_t op); diff --git a/source/libs/function/inc/tscalar.h b/source/libs/function/inc/tscalar.h index da98069c86..f7f30e40c6 100644 --- a/source/libs/function/inc/tscalar.h +++ b/source/libs/function/inc/tscalar.h @@ -19,9 +19,13 @@ extern "C" { #endif -typedef struct SScalarCalcContext { +typedef struct SScalarCtx { + int32_t code; + SSDataBlock *pSrc; + SHashObj *pRes; /* element is SScalarParam */ +} SScalarCtx; -} SScalarCalcContext; +#define SCL_DEFAULT_OP_NUM 10 #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/function/inc/tscalarfunction.h index 6d23775610..f36f5d4fcf 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/function/inc/tscalarfunction.h @@ -21,12 +21,12 @@ extern "C" { #include "function.h" -typedef struct SScalarFuncParam { +typedef struct SScalarParam { void* data; int32_t num; int32_t type; int32_t bytes; -} SScalarFuncParam; +} SScalarParam; typedef struct SScalarFunctionSupport { struct SExprInfo *pExprInfo; @@ -39,7 +39,7 @@ typedef struct SScalarFunctionSupport { extern struct SScalarFunctionInfo scalarFunc[8]; -int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, +int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); diff --git a/source/libs/function/inc/tunaryoperator.h b/source/libs/function/inc/tunaryoperator.h index 27784e8487..08cc6f69c6 100644 --- a/source/libs/function/inc/tunaryoperator.h +++ b/source/libs/function/inc/tunaryoperator.h @@ -22,7 +22,7 @@ extern "C" { #include "tscalarfunction.h" -typedef void (*_unary_scalar_fn_t)(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput); +typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput); _unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 881845f42b..441b6fe3e7 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3240,7 +3240,7 @@ static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; - SScalarFuncParam output = {0}; + SScalarParam output = {0}; output.data = pCtx->pOutput; //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c index 27b5d162a6..31fc2813e2 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/function/src/tbinoperator.c @@ -245,7 +245,7 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { } -int32_t vectorConvertImpl(SScalarFuncParam* pIn, SScalarFuncParam* pOut) { +int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int16_t inType = pIn->type; int16_t inBytes = pIn->bytes; char *input = pIn->data; @@ -512,13 +512,13 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -int32_t vectorConvert(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, SScalarFuncParam* pLeftOut, SScalarFuncParam* pRightOut) { +int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) { if (pLeft->type == pRight->type) { return TSDB_CODE_SUCCESS; } - SScalarFuncParam *param1 = NULL, *paramOut1 = NULL; - SScalarFuncParam *param2 = NULL, *paramOut2 = NULL; + SScalarParam *param1 = NULL, *paramOut1 = NULL; + SScalarParam *param2 = NULL, *paramOut2 = NULL; int32_t code = 0; if (pLeft->type < pRight->type) { @@ -575,7 +575,7 @@ int32_t vectorConvert(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, SScalar return TSDB_CODE_SUCCESS; } -void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -614,7 +614,7 @@ void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int } } -void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -651,7 +651,7 @@ void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int } } } -void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -690,7 +690,7 @@ void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out } } -void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -736,7 +736,7 @@ void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } } -void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -808,7 +808,7 @@ void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *ou } } -void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t len = pLeft->bytes + pRight->bytes; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; @@ -859,7 +859,7 @@ void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } -void vectorBitAnd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -898,7 +898,7 @@ void vectorBitAnd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } } -void vectorBitOr(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -938,7 +938,7 @@ void vectorBitOr(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, i } -void vectorCompareImpl(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int8_t funcIdx = filterGetCompFuncIdx(pLeft->type, optr); @@ -993,14 +993,14 @@ void vectorCompareImpl(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void * } } -void vectorCompare(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord, int32_t optr) { - SScalarFuncParam pLeftOut = {0}; - SScalarFuncParam pRightOut = {0}; +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { + SScalarParam pLeftOut = {0}; + SScalarParam pRightOut = {0}; vectorConvert(pLeft, pRight, &pLeftOut, &pRightOut); - SScalarFuncParam *param1 = NULL; - SScalarFuncParam *param2 = NULL; + SScalarParam *param1 = NULL; + SScalarParam *param2 = NULL; int32_t type = 0; if (pLeftOut->type) { @@ -1018,55 +1018,55 @@ void vectorCompare(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, vectorCompareImpl(pLeftOut, pRightOut, out, _ord, TSDB_RELATION_GREATER); } -void vectorGreater(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER); } -void vectorGreaterEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_GREATER_EQUAL); } -void vectorLower(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS); } -void vectorLowerEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LESS_EQUAL); } -void vectorEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_EQUAL); } -void vectorNotEqual(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_EQUAL); } -void vectorIn(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_IN); } -void vectorNotIn(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_IN); } -void vectorLike(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_LIKE); } -void vectorNotLike(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NOT_LIKE); } -void vectorMatch(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_MATCH); } -void vectorNotMatch(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { vectorCompare(pLeft, pRight, out, _ord, TSDB_RELATION_NMATCH); } -void vectorIsNull(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; @@ -1086,7 +1086,7 @@ void vectorIsNull(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, } } -void vectorNotNull(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { +void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; @@ -1143,9 +1143,9 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorMatch; case OP_TYPE_NMATCH: return vectorNotMatch; - case OP_TYPE_ISNULL: + case OP_TYPE_IS_NULL: return vectorIsNull; - case OP_TYPE_NOTNULL: + case OP_TYPE_IS_NOT_NULL: return vectorNotNull; case OP_TYPE_BIT_AND: return vectorBitAnd; diff --git a/source/libs/function/src/tscalar.c b/source/libs/function/src/tscalar.c index e374ce2734..df61d628b3 100644 --- a/source/libs/function/src/tscalar.c +++ b/source/libs/function/src/tscalar.c @@ -2,79 +2,326 @@ #include "tscalar.h" int32_t sclGetOperatorParamNum(EOperatorType type) { - if (OP_TYPE_ISNULL == type || OP_TYPE_NOTNULL == type) { + if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) { return 1; } return 2; } -int32_t sclPrepareFunctionParams(SScalarFuncParam **pParams, SNodeList* pParameterList) { - *pParams = calloc(pParameterList->length, sizeof(SScalarFuncParam)); - if (NULL == *pParams) { - sclError("calloc %d failed", pParameterList->length * sizeof(SScalarFuncParam)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; +void sclFreeRes(SHashObj *res) { + SScalarParam *p = NULL; + void *pIter = taosHashIterate(res, NULL); + while (pIter) { + p = (SScalarParam *)pIter; + + if (p) { + tfree(p->data); + } + + pIter = taosHashIterate(res, pIter); + } + + taosHashCleanup(res); +} + +void sclFreeParam(SScalarParam *param) { + tfree(param->data); +} + +int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { + switch (nodeType(node)) { + case QUERY_NODE_VALUE: { + SValueNode *valueNode = (SValueNode *)node; + param->data = nodesGetValueFromNode(valueNode); + param->num = 1; + param->type = valueNode->node.resType.type; + param->bytes = valueNode->node.resType.bytes; + + break; + } + case QUERY_NODE_COLUMN_REF: { + if (NULL == ctx) { + sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SColumnRef *ref = (SColumnRef *)node; + if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { + sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, taosArrayGetSize(ctx->pSrc->pDataBlock)); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); + param->data = columnData->pData; + param->num = ctx->pSrc->info.rows; + param->type = columnData->info.type; + param->bytes = columnData->info.bytes; + + break; + } + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_OPERATOR: { + if (NULL == ctx) { + sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); + if (NULL == res) { + sclError("no result for node, type:%d, node:%p", nodeType(node), node); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *param = *res; + + break; + } } - SListCell *cell = pParameterList->pHead; - for (int32_t i = 0; i < pParameterList->length; ++i) { - if (NULL == cell || NULL == cell->pNode) { - sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); - tfree(*pParams); - return TSDB_CODE_QRY_INVALID_INPUT; + if (param->num > *rowNum) { + if (1 != param->num) && (1 < *rowNum) { + sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->num); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - - if (QUERY_NODE_VALUE != nodeType(cell->pNode)) { - sclError("invalid node type in cell, type:%d", nodeType(cell->pNode)); - tfree(*pParams); - return TSDB_CODE_QRY_APP_ERROR; - } - - SValueNode *valueNode = (SValueNode *)cell->pNode; - pParams[i].data = nodesGetValueFromNode(valueNode); - pParams[i].num = 1; - pParams[i].type = valueNode->node.resType.type; - pParams[i].bytes = valueNode->node.resType.bytes; - - cell = cell->pNext; + + *rowNum = param->num; } return TSDB_CODE_SUCCESS; } -EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { - SFunctionNode *node = (SFunctionNode *)*pNode; +int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { + SScalarParam *param = NULL; + + for (int32_t i = 0; i < num; ++i) { + param = params + i; + + if (1 == param->num) { + continue; + } + + if (IS_VAR_DATA_TYPE(param->type)) { + param->data = (char *)(param->data) + varDataTLen(param->data); + } else { + param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { + int32_t code = 0; + *pParams = calloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == *pParams) { + sclError("calloc %d failed", pParamList->length * sizeof(SScalarParam)); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SListCell *cell = pParamList->pHead; + for (int32_t i = 0; i < pParamList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SCL_ERR_JRET(sclInitParam(cell->pNode, &pParams[i], ctx, rowNum)); + + cell = cell->pNext; + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(*pParams); + SCL_RET(code); +} + +int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) { + int32_t code = 0; + int32_t paramNum = sclGetOperatorParamNum(node->opType); + if (NULL == node->pLeft || (paramNum == 2 && NULL == node->pRight)) { + sclError("invalid operation node, left:%p, right:%p", node->pLeft, node->pRight); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + *pParams = calloc(paramNum, sizeof(SScalarParam)); + if (NULL == *pParams) { + sclError("calloc %d failed", paramNum * sizeof(SScalarParam)); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SCL_ERR_JRET(sclInitParam(node->pLeft, &pParams[0], ctx, rowNum)); + if (paramNum > 1) { + SCL_ERR_JRET(sclInitParam(node->pRight, &pParams[1], ctx, rowNum)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(*pParams); + SCL_RET(code); +} + + +int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { if (NULL == node->pParameterList || node->pParameterList->length <= 0) { sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); - *(int32_t *)pContext = code; - return DEAL_RES_ERROR; + SCL_ERR_RET(code); } - SScalarFuncParam *input = NULL; - if (sclPrepareFunctionParams(&input, node->pParameterList)) { - return DEAL_RES_ERROR; + SScalarParam *params = NULL; + int32_t rowNum = 0; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)rowNum * sizeof(tDataTypes[output->type].bytes)); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SScalarFuncParam output = {0}; + for (int32_t i = 0; i < rowNum; ++i) { + code = (*ffpSet.process)(params, node->pParameterList->length, output); + if (code) { + sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } + + sclParamMoveNext(output, 1); + sclParamMoveNext(params, node->pParameterList->length); + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + SCL_RET(code); +} + + +int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *output) { + if (NULL == node->pParameterList || node->pParameterList->length <= 0) { + sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (TSDB_DATA_TYPE_BOOL != node->node.resType.type) { + sclError("invalid logic resType, type:%d", node->node.resType.type); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (LOGIC_COND_TYPE_NOT == node->condType && node->pParameterList->length > 1) { + sclError("invalid NOT operation parameter number, paramNum:%d", node->pParameterList->length); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + SScalarParam *params = NULL; + int32_t rowNum = 0; + int32_t code = 0; - code = (*ffpSet.process)(input, node->pParameterList->length, &output); - if (code) { - sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); - *(int32_t *)pContext = code; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, sizeof(bool)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)rowNum * sizeof(bool)); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + bool value = false; + + for (int32_t i = 0; i < rowNum; ++i) { + for (int32_t m = 0; m < node->pParameterList->length; ++m) { + GET_TYPED_DATA(value, bool, params[m].type, params[m].data); + + if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + break; + } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + break; + } else if (LOGIC_COND_TYPE_NOT == node->condType) { + value = !value; + } + } + + *(bool *)output->data = value; + + sclParamMoveNext(output, 1); + sclParamMoveNext(params, node->pParameterList->length); + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + CTG_RET(code); +} + +int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *output) { + SScalarParam *params = NULL; + int32_t rowNum = 0; + int32_t code = 0; + + SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); + + output->type = node->node.resType.type; + output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + if (NULL == output->data) { + sclError("calloc %d failed", (int32_t)rowNum * sizeof(tDataTypes[output->type].bytes)); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); + + int32_t paramNum = sclGetOperatorParamNum(node->opType); + SScalarParam* pLeft = ¶ms[0]; + SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; + + for (int32_t i = 0; i < rowNum; ++i) { + + OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); + + sclParamMoveNext(output, 1); + sclParamMoveNext(pLeft, 1); + if (pRight) { + sclParamMoveNext(pRight, 1); + } + } + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(params); + CTG_RET(code); +} + + +EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { + SFunctionNode *node = (SFunctionNode *)*pNode; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecFuncion(node, NULL, &output); + if (*(int32_t *)pContext) { return DEAL_RES_ERROR; } SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); + sclFreeParam(&output); *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -86,107 +333,66 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { nodesDestroyNode(*pNode); *pNode = (SNode*)res; - tfree(output.data); + sclFreeParam(&output); return DEAL_RES_CONTINUE; } EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { SLogicConditionNode *node = (SLogicConditionNode *)*pNode; - if (NULL == node->pParameterList || node->pParameterList->length <= 0) { - sclError("invalid logic parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + SScalarParam output = {0}; + + *(int32_t *)pContext = sclExecLogic(node, NULL, &output); + if (*(int32_t *)pContext) { return DEAL_RES_ERROR; } - if (LOGIC_COND_TYPE_NOT == node->condType && node->pParameterList->length > 1) { - sclError("invalid NOT operation parameter number, paramNum:%d", node->pParameterList->length); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } - - bool value = false; - SListCell *cell = node->pParameterList->pHead; - for (int32_t i = 0; i < node->pParameterList->length; ++i) { - if (NULL == cell || NULL == cell->pNode) { - sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); - return TSDB_CODE_QRY_INVALID_INPUT; - } - - if (QUERY_NODE_VALUE != nodeType(cell->pNode)) { - sclError("invalid node type in cell, type:%d", nodeType(cell->pNode)); - return TSDB_CODE_QRY_APP_ERROR; - } - - SValueNode *valueNode = (SValueNode *)cell->pNode; - GET_TYPED_DATA(value, bool, valueNode->node.resType.type, nodesGetValueFromNode(valueNode)); - if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { - break; - } else if (LOGIC_COND_TYPE_OR == node->condType && value) { - break; - } else if (LOGIC_COND_TYPE_NOT == node->condType) { - value = !value; - } - - cell = cell->pNext; - } - SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); + sclFreeParam(&output); *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } res->node.resType = node->node.resType; - SET_TYPED_DATA(nodesGetValueFromNode(res), res->node.resType.type, value); + SET_TYPED_DATA(nodesGetValueFromNode(res), res->node.resType.type, output.data); nodesDestroyNode(*pNode); *pNode = (SNode*)res; + sclFreeParam(&output); + return DEAL_RES_CONTINUE; } EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { - SOperatorNode *oper = (SOperatorNode *)*pNode; - int32_t paramNum = sclGetOperatorParamNum(oper->opType); - if (NULL == oper->pLeft || (paramNum == 2 && NULL == oper->pRight)) { - sclError("invalid operation node, left:%p, right:%p", oper->pLeft, oper->pRight); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } + SOperatorNode *node = (SOperatorNode *)*pNode; + SScalarParam output = {0}; - if (QUERY_NODE_VALUE != nodeType(oper->pLeft) || (paramNum == 2 && QUERY_NODE_VALUE != nodeType(oper->pRight))) { - sclError("invalid operation node, leftType:%d, rightType:%d", nodeType(oper->pLeft), oper->pRight ? nodeType(oper->pRight) : 0); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + *(int32_t *)pContext = sclExecOperator(node, NULL, &output); + if (*(int32_t *)pContext) { return DEAL_RES_ERROR; } SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { - sclError("make value node failed"); + sclError("make value node failed"); + sclFreeParam(&output); *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } - res->node.resType = oper->node.resType; + res->node.resType = node->node.resType; - SValueNode *leftValue = (SValueNode *)oper->pLeft; - SValueNode *rightValue = (SValueNode *)oper->pRight; - - SScalarFuncParam leftParam = {0}, rightParam = {0}; - _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(oper->opType); - setScalarFuncParam(&leftParam, leftValue->node.resType.type, 0, nodesGetValueFromNode(leftValue), 1); - if (2 == paramNum) { - setScalarFuncParam(&rightParam, rightValue->node.resType.type, 0, nodesGetValueFromNode(rightValue), 1); - } - - OperatorFn(&leftParam, &rightParam, nodesGetValueFromNode(res), TSDB_ORDER_ASC); + SET_TYPED_DATA(nodesGetValueFromNode(res), res->node.resType.type, output.data); nodesDestroyNode(*pNode); *pNode = (SNode*)res; + sclFreeParam(&output); + return DEAL_RES_CONTINUE; } @@ -204,73 +410,99 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { return sclRewriteLogic(pNode, pContext); } - if (QUERY_NODE_OPERATOR != nodeType(*pNode)) { - sclError("invalid node type for calculating constants, type:%d", ); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclRewriteOperator(pNode, pContext); } - return sclRewriteOperator(pNode, pContext); + sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); + + *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; } -EDealRes sclCalculate(SNode** pNode, void* pContext) { + +EDealRes sclWalkFunction(SNode** pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SFunctionNode *node = (SFunctionNode *)*pNode; + SScalarParam output = {0}; + + ctx->code = sclExecFuncion(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclWalkLogic(SNode** pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SScalarParam output = {0}; + + ctx->code = sclExecLogic(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclWalkOperator(SNode** pNode, void* pContext) { + SScalarCtx *ctx = (SScalarCtx *)pContext; + SOperatorNode *node = (SOperatorNode *)*pNode; + SScalarParam output = {0}; + + ctx->code = sclExecOperator(node, ctx, &output); + if (ctx->code) { + return DEAL_RES_ERROR; + } + + if (taosHashPut(ctx->pRes, pNode, POINTER_BYTES, &output, sizeof(output))) { + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + return DEAL_RES_CONTINUE; +} + + +EDealRes sclCalcWalker(SNode** pNode, void* pContext) { if (QUERY_NODE_VALUE == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - return sclCalculateFunction(pNode, pContext); + return sclWalkFunction(pNode, pContext); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { - return sclCalculateLogic(pNode, pContext); + return sclWalkLogic(pNode, pContext); } - if (QUERY_NODE_OPERATOR != nodeType(*pNode)) { - sclError("invalid node type for calculating constants, type:%d", ); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + return sclWalkOperator(pNode, pContext); + } + + sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); + + SScalarCtx *ctx = (SScalarCtx *)pContext; - SOperatorNode *oper = (SOperatorNode *)*pNode; - int32_t paramNum = sclGetOperatorParamNum(oper->opType); - if (NULL == oper->pLeft || (paramNum == 2 && NULL == oper->pRight)) { - sclError("invalid operation node, left:%p, right:%p", oper->pLeft, oper->pRight); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } - - if (QUERY_NODE_VALUE != nodeType(oper->pLeft) || (paramNum == 2 && QUERY_NODE_VALUE != nodeType(oper->pRight))) { - sclError("invalid operation node, leftType:%d, rightType:%d", nodeType(oper->pLeft), oper->pRight ? nodeType(oper->pRight) : 0); - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; - } - - SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { - sclError("make value node failed"); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; - return DEAL_RES_ERROR; - } - - res->node.resType = oper->node.resType; - - SValueNode *leftValue = (SValueNode *)oper->pLeft; - SValueNode *rightValue = (SValueNode *)oper->pRight; - - SScalarFuncParam leftParam = {0}, rightParam = {0}; - _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(oper->opType); - setScalarFuncParam(&leftParam, leftValue->node.resType.type, 0, nodesGetValueFromNode(leftValue), 1); - if (2 == paramNum) { - setScalarFuncParam(&rightParam, rightValue->node.resType.type, 0, nodesGetValueFromNode(rightValue), 1); - } + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; - OperatorFn(&leftParam, &rightParam, nodesGetValueFromNode(res), TSDB_ORDER_ASC); - - nodesDestroyNode(*pNode); - *pNode = (SNode*)res; - - return DEAL_RES_CONTINUE; + return DEAL_RES_ERROR; } @@ -294,23 +526,39 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { SCL_RET(code); } -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SSDataBlock *pDst) { - if (NULL == pNode) { +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { + if (NULL == pNode || NULL == pSrc || NULL == pDst) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; + SScalarCtx ctx = {.code = 0, .pSrc = pSrc}; - nodesRewriteNodePostOrder(&pNode, sclCalculate, (void *)&code); + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + nodesWalkNodePostOrder(&pNode, sclCalcWalker, (void *)&ctx); - if (code) { + if (ctx.code) { nodesDestroyNode(pNode); - SCL_ERR_RET(code); + sclFreeRes(ctx.pRes); + SCL_ERR_RET(ctx.code); } - *pRes = pNode; + SScalarParam *res = taosHashGet(ctx.pRes, &pNode, POINTER_BYTES); + if (NULL == res) { + sclError("no res for calculating, node:%d, type:%d", pNode, nodeType(pNode)); + SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + *pDst = *res; - SCL_RET(code); + nodesDestroyNode(pNode); + + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 50aaa07757..df8a654c96 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -2,13 +2,13 @@ #include "tbinoperator.h" #include "tunaryoperator.h" -static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { +static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { dst->type = src->type; dst->bytes = src->bytes; dst->num = src->num; } -static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tceil(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -34,7 +34,7 @@ static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun } } -static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tfloor(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -62,7 +62,7 @@ static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu } } -static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void _tabs(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -120,7 +120,7 @@ static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFun } } -static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tround(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); assert(numOfInput == 1); @@ -146,7 +146,7 @@ static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFu } } -static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tlength(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assert(numOfInput == 1); int64_t* out = (int64_t*) pOutput->data; @@ -157,7 +157,7 @@ static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF } } -static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tconcat(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { assert(numOfInput > 0); int32_t rowLen = 0; @@ -189,11 +189,11 @@ static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarF } } -static void tltrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void tltrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { } -static void trtrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { +static void trtrim(SScalarParam* pOutput, size_t numOfInput, const SScalarParam *pLeft) { } @@ -262,7 +262,7 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf } } -static void setScalarFuncParam(SScalarFuncParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { +static void setScalarFuncParam(SScalarParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { param->bytes = bytes; param->type = type; param->num = numOfRows; From 8f199e0eb89291ddea340ca285583e7f10600797 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 17 Feb 2022 19:30:43 +0800 Subject: [PATCH 4/9] feature/qnode --- include/common/tname.h | 10 + include/common/ttypes.h | 2 - include/libs/function/function.h | 11 +- include/libs/function/functionMgt.h | 1 + include/libs/nodes/nodes.h | 1 - include/libs/nodes/querynodes.h | 1 + include/libs/scalar/scalar.h | 36 ++ source/libs/CMakeLists.txt | 3 +- source/libs/executor/CMakeLists.txt | 2 +- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 6 +- source/libs/executor/test/CMakeLists.txt | 2 +- source/libs/executor/test/executorTests.cpp | 2 + source/libs/function/inc/tunaryoperator.h | 6 +- source/libs/function/src/taggfunction.c | 5 +- source/libs/function/src/tfunction.c | 3 +- source/libs/function/src/tunaryoperator.c | 12 +- source/libs/nodes/src/nodesUtilFuncs.c | 1 + source/libs/scalar/CMakeLists.txt | 16 + source/libs/scalar/inc/filter.h | 347 ++++++++++++++++++ .../inc/tscalar.h => scalar/inc/sclInt.h} | 9 +- .../inc/sclfunc.h} | 8 +- .../tbinoperator.h => scalar/inc/sclvector.h} | 2 +- .../libs/{executor => scalar}/inc/tfilter.h | 12 +- source/libs/scalar/inc/tsclfunc.h | 45 +++ .../src/tfilter.c => scalar/src/filter.c} | 6 +- .../src/tscalar.c => scalar/src/scalar.c} | 114 +++--- .../src/sclfunc.c} | 5 +- .../tbinoperator.c => scalar/src/sclvector.c} | 30 +- source/libs/scalar/test/CMakeLists.txt | 18 + source/libs/scalar/test/scalarTests.cpp | 55 +++ 31 files changed, 670 insertions(+), 103 deletions(-) create mode 100644 include/libs/scalar/scalar.h create mode 100644 source/libs/scalar/CMakeLists.txt create mode 100644 source/libs/scalar/inc/filter.h rename source/libs/{function/inc/tscalar.h => scalar/inc/sclInt.h} (91%) rename source/libs/{function/inc/tscalarfunction.h => scalar/inc/sclfunc.h} (92%) rename source/libs/{function/inc/tbinoperator.h => scalar/inc/sclvector.h} (97%) rename source/libs/{executor => scalar}/inc/tfilter.h (97%) create mode 100644 source/libs/scalar/inc/tsclfunc.h rename source/libs/{executor/src/tfilter.c => scalar/src/filter.c} (99%) rename source/libs/{function/src/tscalar.c => scalar/src/scalar.c} (80%) rename source/libs/{function/src/tscalarfunction.c => scalar/src/sclfunc.c} (99%) rename source/libs/{function/src/tbinoperator.c => scalar/src/sclvector.c} (98%) create mode 100644 source/libs/scalar/test/CMakeLists.txt create mode 100644 source/libs/scalar/test/scalarTests.cpp diff --git a/include/common/tname.h b/include/common/tname.h index 12a0d34cb4..47028fbce1 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,6 +16,11 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H +#ifdef __cplusplus +extern "C" { +#endif + + #include "tdef.h" #include "tmsg.h" @@ -59,4 +64,9 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId); SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name); +#ifdef __cplusplus +} +#endif + + #endif // TDENGINE_TNAME_H diff --git a/include/common/ttypes.h b/include/common/ttypes.h index afa605044a..5a1e442277 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -122,8 +122,6 @@ typedef struct { *(int32_t *)(_v) = (int32_t)(_data); \ break; \ default: \ - (void *)(_v) = (void *)(_data); \ - (void *)(_data) = NULL; \ break; \ } \ } while (0) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index dfcac7ca3f..bb5d51686d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -226,7 +226,12 @@ typedef struct SAggFunctionInfo { int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; -struct SScalarParam; +typedef struct SScalarParam { + void* data; + int32_t num; + int32_t type; + int32_t bytes; +} SScalarParam; typedef struct SScalarFunctionInfo { char name[FUNCTIONS_NAME_MAX_LENGTH]; @@ -285,10 +290,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); -struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num); -void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num); -struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index); - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // fill api struct SFillInfo; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 84b1f8d5bf..e77ccaa225 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "querynodes.h" +#include "function.h" typedef enum EFunctionType { // aggregate function diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 8a8e230359..068b80cf59 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -104,7 +104,6 @@ SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SNode* nodesListGetNode(SNodeList* pList, int32_t index); void nodesDestroyList(SNodeList* pList); -void *nodesGetValueFromNode(SValueNode *pNode); typedef enum EDealRes { DEAL_RES_CONTINUE = 1, diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index cea0aa32a8..de6f8747e8 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -277,6 +277,7 @@ bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); +void *nodesGetValueFromNode(SValueNode *pNode); #ifdef __cplusplus } diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h new file mode 100644 index 0000000000..df83a0de35 --- /dev/null +++ b/include/libs/scalar/scalar.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_SCALAR_H +#define TDENGINE_SCALAR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "function.h" +#include "nodes.h" + +typedef struct SFilterInfo SFilterInfo; + + +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_SCALAR_H diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 049b69991f..a2454e4c5e 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -13,4 +13,5 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) -add_subdirectory(nodes) \ No newline at end of file +add_subdirectory(nodes) +add_subdirectory(scalar) \ No newline at end of file diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 9b53cc1fbb..12a78134c3 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom vnode + PRIVATE os util common function parser planner qcom vnode scalar nodes ) target_include_directories( diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 232b54554f..23acdf1c33 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -29,9 +29,9 @@ extern "C" { #include "executil.h" #include "executor.h" #include "planner.h" +#include "scalar.h" #include "taosdef.h" #include "tarray.h" -#include "tfilter.h" #include "thash.h" #include "tlockfree.h" #include "tpagedbuf.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f5dc7a82b1..5508b5ecd7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2148,7 +2148,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); - pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); + //pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { @@ -2174,7 +2174,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT return TSDB_CODE_SUCCESS; _clean: - destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); + //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); tfree(pRuntimeEnv->pResultRowHashTable); tfree(pRuntimeEnv->keyBuf); tfree(pRuntimeEnv->prevRow); @@ -2212,7 +2212,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); - destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); + //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); // destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt index ece84207c7..c24993eb89 100644 --- a/source/libs/executor/test/CMakeLists.txt +++ b/source/libs/executor/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( executorTest - PUBLIC os util common transport gtest taos qcom executor function planner + PRIVATE os util common transport gtest taos qcom executor function planner scalar nodes ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index ebea6755d7..c9657607c2 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -32,6 +32,8 @@ #include "trpc.h" #include "stub.h" #include "executor.h" +#include "tmsg.h" +#include "tname.h" namespace { diff --git a/source/libs/function/inc/tunaryoperator.h b/source/libs/function/inc/tunaryoperator.h index 08cc6f69c6..cd40297e07 100644 --- a/source/libs/function/inc/tunaryoperator.h +++ b/source/libs/function/inc/tunaryoperator.h @@ -20,10 +20,10 @@ extern "C" { #endif -#include "tscalarfunction.h" +//#include "tscalarfunction.h" -typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput); -_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); +//typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput); +//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 441b6fe3e7..1aa5871468 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "tscalarfunction.h" #include "os.h" #include "taosdef.h" #include "tmsg.h" @@ -3221,6 +3220,7 @@ static void diff_function(SqlFunctionCtx *pCtx) { } } +#if 0 char *getArithColumnData(void *param, const char* name, int32_t colId) { SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; @@ -3235,10 +3235,11 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { assert(index >= 0); return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } +#endif static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; + //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; SScalarParam output = {0}; output.data = pCtx->pOutput; diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 36c9e2513f..e302643c32 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -3,7 +3,6 @@ #include "function.h" #include "thash.h" #include "taggfunction.h" -#include "tscalarfunction.h" static SHashObj* functionHashTable = NULL; static SHashObj* udfHashTable = NULL; @@ -18,12 +17,14 @@ static void doInitFunctionHashTable() { taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } +/* numOfEntries = tListLen(scalarFunc); for(int32_t i = 0; i < numOfEntries; ++i) { int32_t len = (int32_t) strlen(scalarFunc[i].name); SScalarFunctionInfo* ptr = &scalarFunc[i]; taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } +*/ udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true); } diff --git a/source/libs/function/src/tunaryoperator.c b/source/libs/function/src/tunaryoperator.c index 9651f98d08..957f0799c5 100644 --- a/source/libs/function/src/tunaryoperator.c +++ b/source/libs/function/src/tunaryoperator.c @@ -4,10 +4,10 @@ // TODO dynamic define these functions -_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { - assert(0); -} +//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { +// assert(0); +//} -bool isStringOperatorFn(int32_t op) { - return op == FUNCTION_LENGTH; -} +//bool isStringOperatorFn(int32_t op) { +// return op == FUNCTION_LENGTH; +//} diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 8eef6aa364..520a3f82ca 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -16,6 +16,7 @@ #include "querynodes.h" #include "nodesShowStmts.h" #include "taoserror.h" +#include "taos.h" static SNode* makeNode(ENodeType type, size_t size) { SNode* p = calloc(1, size); diff --git a/source/libs/scalar/CMakeLists.txt b/source/libs/scalar/CMakeLists.txt new file mode 100644 index 0000000000..d2d02bc0dc --- /dev/null +++ b/source/libs/scalar/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src SCALAR_SRC) + +add_library(scalar STATIC ${SCALAR_SRC}) +target_include_directories( + scalar + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries(scalar + PRIVATE os util common nodes function qcom + ) + +if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/scalar/inc/filter.h b/source/libs/scalar/inc/filter.h new file mode 100644 index 0000000000..27665da0c8 --- /dev/null +++ b/source/libs/scalar/inc/filter.h @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_QFILTER_H +#define TDENGINE_QFILTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "thash.h" +#include "tname.h" +#include "common.h" +#include "scalar.h" +#include "querynodes.h" + +#define FILTER_DEFAULT_GROUP_SIZE 4 +#define FILTER_DEFAULT_UNIT_SIZE 4 +#define FILTER_DEFAULT_FIELD_SIZE 4 +#define FILTER_DEFAULT_VALUE_SIZE 4 +#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2 + +#define FILTER_DUMMY_EMPTY_OPTR 127 + +#define MAX_NUM_STR_SIZE 40 + +#define FILTER_RM_UNIT_MIN_ROWS 100 + +enum { + FLD_TYPE_COLUMN = 1, + FLD_TYPE_VALUE = 2, + FLD_TYPE_MAX = 3, + FLD_DESC_NO_FREE = 4, + FLD_DATA_NO_FREE = 8, + FLD_DATA_IS_HASH = 16, +}; + +enum { + MR_ST_START = 1, + MR_ST_FIN = 2, + MR_ST_ALL = 4, + MR_ST_EMPTY = 8, +}; + +enum { + RANGE_FLG_EXCLUDE = 1, + RANGE_FLG_INCLUDE = 2, + RANGE_FLG_NULL = 4, +}; + +enum { + FI_OPTION_NO_REWRITE = 1, + FI_OPTION_TIMESTAMP = 2, + FI_OPTION_NEED_UNIQE = 4, +}; + +enum { + FI_STATUS_ALL = 1, + FI_STATUS_EMPTY = 2, + FI_STATUS_REWRITE = 4, + FI_STATUS_CLONED = 8, +}; + +enum { + FI_STATUS_BLK_ALL = 1, + FI_STATUS_BLK_EMPTY = 2, + FI_STATUS_BLK_ACTIVE = 4, +}; + +enum { + RANGE_TYPE_UNIT = 1, + RANGE_TYPE_VAR_HASH = 2, + RANGE_TYPE_MR_CTX = 3, +}; + +typedef struct OptrStr { + uint16_t optr; + char *str; +} OptrStr; + +typedef struct SFilterRange { + int64_t s; + int64_t e; + char sflag; + char eflag; +} SFilterRange; + +typedef struct SFilterColRange { + uint16_t idx; //column field idx + bool isNull; + bool notNull; + bool isRange; + SFilterRange ra; +} SFilterColRange; + +typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); +typedef int32_t(*filter_desc_compare_func)(const void *, const void *); +typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); + +typedef struct SFilterRangeCompare { + int64_t s; + int64_t e; + rangeCompFunc func; +} SFilterRangeCompare; + +typedef struct SFilterRangeNode { + struct SFilterRangeNode* prev; + struct SFilterRangeNode* next; + union { + SFilterRange ra; + SFilterRangeCompare rc; + }; +} SFilterRangeNode; + +typedef struct SFilterRangeCtx { + int32_t type; + int32_t options; + int8_t status; + bool isnull; + bool notnull; + bool isrange; + int16_t colId; + __compar_fn_t pCompareFunc; + SFilterRangeNode *rf; //freed + SFilterRangeNode *rs; +} SFilterRangeCtx ; + +typedef struct SFilterVarCtx { + int32_t type; + int32_t options; + int8_t status; + bool isnull; + bool notnull; + bool isrange; + SHashObj *wild; + SHashObj *value; +} SFilterVarCtx; + +typedef struct SFilterField { + uint16_t flag; + void* desc; + void* data; +} SFilterField; + +typedef struct SFilterFields { + uint16_t size; + uint16_t num; + SFilterField *fields; +} SFilterFields; + +typedef struct SFilterFieldId { + uint16_t type; + uint16_t idx; +} SFilterFieldId; + +typedef struct SFilterGroup { + uint16_t unitSize; + uint16_t unitNum; + uint16_t *unitIdxs; + uint8_t *unitFlags; // !unit result +} SFilterGroup; + +typedef struct SFilterColInfo { + uint8_t type; + int32_t dataType; + void *info; +} SFilterColInfo; + +typedef struct SFilterGroupCtx { + uint16_t colNum; + uint16_t *colIdx; + SFilterColInfo *colInfo; +} SFilterGroupCtx; + +typedef struct SFilterColCtx { + uint16_t colIdx; + void* ctx; +} SFilterColCtx; + +typedef struct SFilterCompare { + uint8_t type; + uint8_t optr; + uint8_t optr2; +} SFilterCompare; + +typedef struct SFilterUnit { + SFilterCompare compare; + SFilterFieldId left; + SFilterFieldId right; + SFilterFieldId right2; +} SFilterUnit; + +typedef struct SFilterComUnit { + void *colData; + void *valData; + void *valData2; + uint16_t colId; + uint16_t dataSize; + uint8_t dataType; + uint8_t optr; + int8_t func; + int8_t rfunc; +} SFilterComUnit; + +typedef struct SFilterPCtx { + SHashObj *valHash; + SHashObj *unitHash; +} SFilterPCtx; + +typedef struct SFilterInfo { + uint32_t options; + uint32_t status; + uint16_t unitSize; + uint16_t unitNum; + uint16_t groupNum; + uint16_t colRangeNum; + SFilterFields fields[FLD_TYPE_MAX]; + SFilterGroup *groups; + uint16_t *cgroups; + SFilterUnit *units; + SFilterComUnit *cunits; + uint8_t *unitRes; // result + uint8_t *unitFlags; // got result + SFilterRangeCtx **colRange; + filter_exec_func func; + uint8_t blkFlag; + uint16_t blkGroupNum; + uint16_t *blkUnits; + int8_t *blkUnitRes; + + SFilterPCtx pctx; +} SFilterInfo; + +#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) + +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) +#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) + +#define MR_EMPTY_RES(ctx) (ctx->rs == NULL) + +#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) +#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) +#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) +#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) + + +#define FILTER_GET_FLAG(st, f) (st & f) +#define FILTER_SET_FLAG(st, f) st |= (f) +#define FILTER_CLR_FLAG(st, f) st &= (~f) + +#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) +#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) +#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) +#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) + +#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0) +#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0) +#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0) +#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) +#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) + +#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) +#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) +#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) + +#define CHK_RETV(c) do { if (c) { return; } } while (0) +#define CHK_RET(c, r) do { if (c) { return r; } } while (0) +#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) +#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) +#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) + +#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) +#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) +#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) +#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) +#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) + +#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) +#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left) +#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right) +#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2) +#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type) +#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u)) +#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri) +#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) +#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) +#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) +#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) +#define FILTER_UNIT_OPTR(u) ((u)->compare.optr) +#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) + +#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags)) +#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1 +#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx]) +#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx]) +#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v) + +#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0) +#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) +#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) + +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) + +#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) + + +#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) +#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) + +#if 0 +extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); +extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); +extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); +extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); +extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); +extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); +extern void filterFreeInfo(SFilterInfo *info); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); +#else +//REMOVE THESE!!!!!!!!!!!!!!!!!!!! +#include "function.h" +#endif +extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); +extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_QFILTER_H diff --git a/source/libs/function/inc/tscalar.h b/source/libs/scalar/inc/sclInt.h similarity index 91% rename from source/libs/function/inc/tscalar.h rename to source/libs/scalar/inc/sclInt.h index f7f30e40c6..3cfea6890f 100644 --- a/source/libs/function/inc/tscalar.h +++ b/source/libs/scalar/inc/sclInt.h @@ -12,12 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_SCALAR_H -#define TDENGINE_SCALAR_H +#ifndef TDENGINE_SCALARINT_H +#define TDENGINE_SCALARINT_H #ifdef __cplusplus extern "C" { #endif +#include "common.h" +#include "thash.h" +#include "query.h" typedef struct SScalarCtx { int32_t code; @@ -45,4 +48,4 @@ typedef struct SScalarCtx { } #endif -#endif // TDENGINE_SCALAR_H +#endif // TDENGINE_SCALARINT_H \ No newline at end of file diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/scalar/inc/sclfunc.h similarity index 92% rename from source/libs/function/inc/tscalarfunction.h rename to source/libs/scalar/inc/sclfunc.h index f36f5d4fcf..8e03470033 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/scalar/inc/sclfunc.h @@ -20,13 +20,7 @@ extern "C" { #endif #include "function.h" - -typedef struct SScalarParam { - void* data; - int32_t num; - int32_t type; - int32_t bytes; -} SScalarParam; +#include "scalar.h" typedef struct SScalarFunctionSupport { struct SExprInfo *pExprInfo; diff --git a/source/libs/function/inc/tbinoperator.h b/source/libs/scalar/inc/sclvector.h similarity index 97% rename from source/libs/function/inc/tbinoperator.h rename to source/libs/scalar/inc/sclvector.h index 678d6169ab..69800a54ea 100644 --- a/source/libs/function/inc/tbinoperator.h +++ b/source/libs/scalar/inc/sclvector.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "tscalarfunction.h" +#include "sclfunc.h" typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); diff --git a/source/libs/executor/inc/tfilter.h b/source/libs/scalar/inc/tfilter.h similarity index 97% rename from source/libs/executor/inc/tfilter.h rename to source/libs/scalar/inc/tfilter.h index 55edf27949..27665da0c8 100644 --- a/source/libs/executor/inc/tfilter.h +++ b/source/libs/scalar/inc/tfilter.h @@ -22,7 +22,9 @@ extern "C" { #include "thash.h" #include "tname.h" -#include "function.h" +#include "common.h" +#include "scalar.h" +#include "querynodes.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -322,7 +324,7 @@ typedef struct SFilterInfo { #define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) - +#if 0 extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); @@ -331,6 +333,12 @@ extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); +#else +//REMOVE THESE!!!!!!!!!!!!!!!!!!!! +#include "function.h" +#endif +extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); +extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); #ifdef __cplusplus } diff --git a/source/libs/scalar/inc/tsclfunc.h b/source/libs/scalar/inc/tsclfunc.h new file mode 100644 index 0000000000..8e03470033 --- /dev/null +++ b/source/libs/scalar/inc/tsclfunc.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_TSCALARFUNCTION_H +#define TDENGINE_TSCALARFUNCTION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "function.h" +#include "scalar.h" + +typedef struct SScalarFunctionSupport { + struct SExprInfo *pExprInfo; + int32_t numOfCols; + SColumnInfo *colList; + void *exprList; // client side used + int32_t offset; + char** data; +} SScalarFunctionSupport; + +extern struct SScalarFunctionInfo scalarFunc[8]; + +int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput, + void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); + + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSCALARFUNCTION_H diff --git a/source/libs/executor/src/tfilter.c b/source/libs/scalar/src/filter.c similarity index 99% rename from source/libs/executor/src/tfilter.c rename to source/libs/scalar/src/filter.c index c42260b214..b680ac6778 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/scalar/src/filter.c @@ -17,7 +17,7 @@ #include "thash.h" //#include "queryLog.h" #include "tcompare.h" -#include "tfilter.h" +#include "filter.h" OptrStr gOptrStr[] = { {TSDB_RELATION_INVALID, "invalid"}, @@ -273,6 +273,10 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { return comparFn; } +__compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) { + return gDataCompare[filterGetCompFuncIdx(type, optr)]; +} + static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); diff --git a/source/libs/function/src/tscalar.c b/source/libs/scalar/src/scalar.c similarity index 80% rename from source/libs/function/src/tscalar.c rename to source/libs/scalar/src/scalar.c index df61d628b3..c8783b1402 100644 --- a/source/libs/function/src/tscalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1,5 +1,10 @@ #include "nodes.h" -#include "tscalar.h" +#include "common.h" +#include "querynodes.h" +#include "function.h" +#include "functionMgt.h" +#include "sclvector.h" +#include "sclInt.h" int32_t sclGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) { @@ -48,7 +53,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SColumnRef *ref = (SColumnRef *)node; if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { - sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, taosArrayGetSize(ctx->pSrc->pDataBlock)); + sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -80,7 +85,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } if (param->num > *rowNum) { - if (1 != param->num) && (1 < *rowNum) { + if ((1 != param->num) && (1 < *rowNum)) { sclError("different row nums, rowNum:%d, newRowNum:%d", *rowNum, param->num); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -113,9 +118,9 @@ int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { int32_t code = 0; - *pParams = calloc(pParamList->length, sizeof(SScalarParam)); - if (NULL == *pParams) { - sclError("calloc %d failed", pParamList->length * sizeof(SScalarParam)); + SScalarParam *paramList = calloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == paramList) { + sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -126,16 +131,18 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarC SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - SCL_ERR_JRET(sclInitParam(cell->pNode, &pParams[i], ctx, rowNum)); + SCL_ERR_JRET(sclInitParam(cell->pNode, ¶mList[i], ctx, rowNum)); cell = cell->pNext; } + *pParams = paramList; + return TSDB_CODE_SUCCESS; _return: - tfree(*pParams); + tfree(paramList); SCL_RET(code); } @@ -147,22 +154,24 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - *pParams = calloc(paramNum, sizeof(SScalarParam)); - if (NULL == *pParams) { - sclError("calloc %d failed", paramNum * sizeof(SScalarParam)); + SScalarParam *paramList = calloc(paramNum, sizeof(SScalarParam)); + if (NULL == paramList) { + sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCL_ERR_JRET(sclInitParam(node->pLeft, &pParams[0], ctx, rowNum)); + SCL_ERR_JRET(sclInitParam(node->pLeft, ¶mList[0], ctx, rowNum)); if (paramNum > 1) { - SCL_ERR_JRET(sclInitParam(node->pRight, &pParams[1], ctx, rowNum)); + SCL_ERR_JRET(sclInitParam(node->pRight, ¶mList[1], ctx, rowNum)); } + *pParams = paramList; + return TSDB_CODE_SUCCESS; _return: - tfree(*pParams); + tfree(paramList); SCL_RET(code); } @@ -187,7 +196,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu output->type = node->node.resType.type; output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); if (NULL == output->data) { - sclError("calloc %d failed", (int32_t)rowNum * sizeof(tDataTypes[output->type].bytes)); + sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -236,7 +245,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->type = node->node.resType.type; output->data = calloc(rowNum, sizeof(bool)); if (NULL == output->data) { - sclError("calloc %d failed", (int32_t)rowNum * sizeof(bool)); + sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -266,7 +275,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o _return: tfree(params); - CTG_RET(code); + SCL_RET(code); } int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *output) { @@ -277,9 +286,9 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); output->type = node->node.resType.type; - output->data = calloc(rowNum, sizeof(tDataTypes[output->type].bytes)); + output->data = calloc(rowNum, tDataTypes[output->type].bytes); if (NULL == output->data) { - sclError("calloc %d failed", (int32_t)rowNum * sizeof(tDataTypes[output->type].bytes)); + sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -305,7 +314,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp _return: tfree(params); - CTG_RET(code); + SCL_RET(code); } @@ -318,7 +327,7 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); @@ -328,8 +337,13 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { res->node.resType = node->node.resType; - SET_TYPED_DATA(nodesGetValueFromNode(res), output.type, output.data); - + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } + nodesDestroyNode(*pNode); *pNode = (SNode*)res; @@ -347,7 +361,7 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); @@ -357,7 +371,12 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { res->node.resType = node->node.resType; - SET_TYPED_DATA(nodesGetValueFromNode(res), res->node.resType.type, output.data); + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } nodesDestroyNode(*pNode); *pNode = (SNode*)res; @@ -376,7 +395,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - SValueNode *res = nodesMakeNode(QUERY_NODE_VALUE); + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); @@ -386,7 +405,12 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { res->node.resType = node->node.resType; - SET_TYPED_DATA(nodesGetValueFromNode(res), res->node.resType.type, output.data); + if (IS_VAR_DATA_TYPE(output.type)) { + res->datum.p = output.data; + output.data = NULL; + } else { + memcpy(nodesGetValueFromNode(res), output.data, tDataTypes[output.type].bytes); + } nodesDestroyNode(*pNode); *pNode = (SNode*)res; @@ -422,9 +446,9 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { } -EDealRes sclWalkFunction(SNode** pNode, void* pContext) { +EDealRes sclWalkFunction(SNode* pNode, void* pContext) { SScalarCtx *ctx = (SScalarCtx *)pContext; - SFunctionNode *node = (SFunctionNode *)*pNode; + SFunctionNode *node = (SFunctionNode *)pNode; SScalarParam output = {0}; ctx->code = sclExecFuncion(node, ctx, &output); @@ -432,7 +456,7 @@ EDealRes sclWalkFunction(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - if (taosHashPut(ctx->pRes, pNode, POINTER_BYTES, &output, sizeof(output))) { + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -441,9 +465,9 @@ EDealRes sclWalkFunction(SNode** pNode, void* pContext) { } -EDealRes sclWalkLogic(SNode** pNode, void* pContext) { +EDealRes sclWalkLogic(SNode* pNode, void* pContext) { SScalarCtx *ctx = (SScalarCtx *)pContext; - SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SLogicConditionNode *node = (SLogicConditionNode *)pNode; SScalarParam output = {0}; ctx->code = sclExecLogic(node, ctx, &output); @@ -451,7 +475,7 @@ EDealRes sclWalkLogic(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - if (taosHashPut(ctx->pRes, pNode, POINTER_BYTES, &output, sizeof(output))) { + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -460,9 +484,9 @@ EDealRes sclWalkLogic(SNode** pNode, void* pContext) { } -EDealRes sclWalkOperator(SNode** pNode, void* pContext) { +EDealRes sclWalkOperator(SNode* pNode, void* pContext) { SScalarCtx *ctx = (SScalarCtx *)pContext; - SOperatorNode *node = (SOperatorNode *)*pNode; + SOperatorNode *node = (SOperatorNode *)pNode; SScalarParam output = {0}; ctx->code = sclExecOperator(node, ctx, &output); @@ -470,7 +494,7 @@ EDealRes sclWalkOperator(SNode** pNode, void* pContext) { return DEAL_RES_ERROR; } - if (taosHashPut(ctx->pRes, pNode, POINTER_BYTES, &output, sizeof(output))) { + if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -479,24 +503,24 @@ EDealRes sclWalkOperator(SNode** pNode, void* pContext) { } -EDealRes sclCalcWalker(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode)) { +EDealRes sclCalcWalker(SNode* pNode, void* pContext) { + if (QUERY_NODE_VALUE == nodeType(pNode)) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { return sclWalkFunction(pNode, pContext); } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { return sclWalkLogic(pNode, pContext); } - if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { return sclWalkOperator(pNode, pContext); } - sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); + sclError("invalid node type for calculating constants, type:%d", nodeType(pNode)); SScalarCtx *ctx = (SScalarCtx *)pContext; @@ -540,7 +564,7 @@ int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - nodesWalkNodePostOrder(&pNode, sclCalcWalker, (void *)&ctx); + nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); if (ctx.code) { nodesDestroyNode(pNode); @@ -548,9 +572,9 @@ int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { SCL_ERR_RET(ctx.code); } - SScalarParam *res = taosHashGet(ctx.pRes, &pNode, POINTER_BYTES); + SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (NULL == res) { - sclError("no res for calculating, node:%d, type:%d", pNode, nodeType(pNode)); + sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode)); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/scalar/src/sclfunc.c similarity index 99% rename from source/libs/function/src/tscalarfunction.c rename to source/libs/scalar/src/sclfunc.c index df8a654c96..ab2cc8b056 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1,6 +1,5 @@ -#include "tscalarfunction.h" -#include "tbinoperator.h" -#include "tunaryoperator.h" +#include "sclfunc.h" +#include "sclvector.h" static void assignBasicParaInfo(struct SScalarParam* dst, const struct SScalarParam* src) { dst->type = src->type; diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/scalar/src/sclvector.c similarity index 98% rename from source/libs/function/src/tbinoperator.c rename to source/libs/scalar/src/sclvector.c index 31fc2813e2..07f32c87d8 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/scalar/src/sclvector.c @@ -16,8 +16,11 @@ #include "os.h" #include "ttypes.h" -#include "tbinoperator.h" +#include "sclvector.h" #include "tcompare.h" +#include "querynodes.h" +#include "filter.h" +#include "query.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); @@ -152,8 +155,8 @@ int64_t getVectorBigintValue_FLOAT(void *src, int32_t index) { int64_t getVectorBigintValue_DOUBLE(void *src, int32_t index) { return (int64_t)*((double *)src + index); } -_getDoubleValue_fn_t getVectorBigintValueFn(int32_t srcType) { - _getDoubleValue_fn_t p = NULL; +_getBigintValue_fn_t getVectorBigintValueFn(int32_t srcType) { + _getBigintValue_fn_t p = NULL; if(srcType==TSDB_DATA_TYPE_TINYINT) { p = getVectorBigintValue_TINYINT; }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { @@ -325,7 +328,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { if (len < 0){ qError("castConvert taosUcs4ToMbs error 1"); tfree(tmp); - return; + return TSDB_CODE_QRY_APP_ERROR; } tmp[len] = 0; @@ -391,7 +394,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { if (len < 0){ qError("castConvert taosUcs4ToMbs error 1"); tfree(tmp); - return; + return TSDB_CODE_QRY_APP_ERROR; } tmp[len] = 0; @@ -457,7 +460,7 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { if (len < 0){ qError("castConvert taosUcs4ToMbs error 1"); tfree(tmp); - return; + return TSDB_CODE_QRY_APP_ERROR; } tmp[len] = 0; @@ -560,13 +563,13 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut2->num = param2->num; paramOut2->data = malloc(paramOut2->num * tDataTypes[paramOut2->type].bytes); if (NULL == paramOut2->data) { - tfree(paramOut1->data) + tfree(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } code = vectorConvertImpl(param2, paramOut2); if (code) { - tfree(paramOut1->data) + tfree(paramOut1->data); tfree(paramOut2->data); return code; } @@ -941,8 +944,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - int8_t funcIdx = filterGetCompFuncIdx(pLeft->type, optr); - __compar_fn_t fp = gDataCompare[funcIdx]; + __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr); bool res = false; bool *output=(bool *)out; @@ -1003,19 +1005,19 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t SScalarParam *param2 = NULL; int32_t type = 0; - if (pLeftOut->type) { + if (pLeftOut.type) { param1 = &pLeftOut; } else { param1 = pLeft; } - if (pRightOut->type) { + if (pRightOut.type) { param2 = &pRightOut; } else { param2 = pRight; } - vectorCompareImpl(pLeftOut, pRightOut, out, _ord, TSDB_RELATION_GREATER); + vectorCompareImpl(param1, param2, out, _ord, TSDB_RELATION_GREATER); } void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { @@ -1091,7 +1093,7 @@ void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - bool *output=(bool *)out; + bool *output = (bool *)out; _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); for (; i >= 0 && i < pLeft->num; i += step, output += 1) { diff --git a/source/libs/scalar/test/CMakeLists.txt b/source/libs/scalar/test/CMakeLists.txt new file mode 100644 index 0000000000..a9af1ece30 --- /dev/null +++ b/source/libs/scalar/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scalar unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(scalarTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + scalarTest + PUBLIC os util common gtest qcom function nodes +) + +TARGET_INCLUDE_DIRECTORIES( + scalarTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scalar/inc" +) diff --git a/source/libs/scalar/test/scalarTests.cpp b/source/libs/scalar/test/scalarTests.cpp new file mode 100644 index 0000000000..38fe072f9e --- /dev/null +++ b/source/libs/scalar/test/scalarTests.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "stub.h" +#include "addr_any.h" +#include "scalar.h" + +namespace { + +} + +TEST(scalarTest, func) { + +} + + +int main(int argc, char** argv) { + srand(time(NULL)); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#pragma GCC diagnostic pop From 6d04b954ac53fdfcf6fca97d62d5c33d8438a619 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Feb 2022 08:48:54 +0800 Subject: [PATCH 5/9] feature/qnode --- source/libs/scalar/src/scalar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index c8783b1402..082dd6cfd4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -51,7 +51,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SColumnRef *ref = (SColumnRef *)node; + SColumnRefNode *ref = (SColumnRefNode *)node; if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); From 8d413f09f99360b6d50c8b183406b71407cd54fb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Feb 2022 19:43:36 +0800 Subject: [PATCH 6/9] feature/qnode --- .../libs/scalar/filter.h | 23 +- .../libs/scalar/inc/{filter.h => filterInt.h} | 110 +- source/libs/scalar/inc/tfilter.h | 347 ------ source/libs/scalar/src/filter.c | 1015 +++++++++++------ source/libs/scalar/src/scalar.c | 5 + source/libs/scalar/src/sclvector.c | 2 +- 6 files changed, 744 insertions(+), 758 deletions(-) rename source/libs/scalar/inc/tsclfunc.h => include/libs/scalar/filter.h (54%) rename source/libs/scalar/inc/{filter.h => filterInt.h} (80%) delete mode 100644 source/libs/scalar/inc/tfilter.h diff --git a/source/libs/scalar/inc/tsclfunc.h b/include/libs/scalar/filter.h similarity index 54% rename from source/libs/scalar/inc/tsclfunc.h rename to include/libs/scalar/filter.h index 8e03470033..df83a0de35 100644 --- a/source/libs/scalar/inc/tsclfunc.h +++ b/include/libs/scalar/filter.h @@ -12,34 +12,25 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_TSCALARFUNCTION_H -#define TDENGINE_TSCALARFUNCTION_H +#ifndef TDENGINE_SCALAR_H +#define TDENGINE_SCALAR_H #ifdef __cplusplus extern "C" { #endif #include "function.h" -#include "scalar.h" +#include "nodes.h" -typedef struct SScalarFunctionSupport { - struct SExprInfo *pExprInfo; - int32_t numOfCols; - SColumnInfo *colList; - void *exprList; // client side used - int32_t offset; - char** data; -} SScalarFunctionSupport; +typedef struct SFilterInfo SFilterInfo; -extern struct SScalarFunctionInfo scalarFunc[8]; - -int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput, - void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); +int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); +int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); #ifdef __cplusplus } #endif -#endif // TDENGINE_TSCALARFUNCTION_H +#endif // TDENGINE_SCALAR_H diff --git a/source/libs/scalar/inc/filter.h b/source/libs/scalar/inc/filterInt.h similarity index 80% rename from source/libs/scalar/inc/filter.h rename to source/libs/scalar/inc/filterInt.h index 27665da0c8..a23e698deb 100644 --- a/source/libs/scalar/inc/filter.h +++ b/source/libs/scalar/inc/filterInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_QFILTER_H -#define TDENGINE_QFILTER_H +#ifndef TDENGINE_FILTER_INT_H +#define TDENGINE_FILTER_INT_H #ifdef __cplusplus extern "C" { @@ -85,6 +85,12 @@ enum { RANGE_TYPE_MR_CTX = 3, }; +enum { + FI_ACTION_NO_NEED = 1, + FI_ACTION_CONTINUE, + FI_ACTION_STOP, +}; + typedef struct OptrStr { uint16_t optr; char *str; @@ -97,17 +103,11 @@ typedef struct SFilterRange { char eflag; } SFilterRange; -typedef struct SFilterColRange { - uint16_t idx; //column field idx - bool isNull; - bool notNull; - bool isRange; - SFilterRange ra; -} SFilterColRange; - typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); +typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); +typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { int64_t s; @@ -155,37 +155,39 @@ typedef struct SFilterField { } SFilterField; typedef struct SFilterFields { - uint16_t size; - uint16_t num; + uint32_t size; + uint32_t num; SFilterField *fields; } SFilterFields; typedef struct SFilterFieldId { uint16_t type; - uint16_t idx; + uint32_t idx; } SFilterFieldId; typedef struct SFilterGroup { - uint16_t unitSize; - uint16_t unitNum; - uint16_t *unitIdxs; + uint32_t unitSize; + uint32_t unitNum; + uint32_t *unitIdxs; uint8_t *unitFlags; // !unit result } SFilterGroup; typedef struct SFilterColInfo { uint8_t type; int32_t dataType; + uint8_t optr; // for equal operation in the relation of RELATION_IN + int64_t value; // for equal operation in the relation of RELATION_IN void *info; } SFilterColInfo; typedef struct SFilterGroupCtx { - uint16_t colNum; - uint16_t *colIdx; + uint32_t colNum; + uint32_t *colIdx; SFilterColInfo *colInfo; } SFilterGroupCtx; typedef struct SFilterColCtx { - uint16_t colIdx; + uint32_t colIdx; void* ctx; } SFilterColCtx; @@ -219,16 +221,31 @@ typedef struct SFilterPCtx { SHashObj *unitHash; } SFilterPCtx; +typedef struct SFltTreeStat { + int32_t code; +} SFltTreeStat; + +typedef struct SFltScalarCtx { + SNode *node; +} SFltScalarCtx; + +typedef struct SFltBuildGroupCtx { + SFilterInfo *info; + SArray *group; + int32_t code; +} SFltBuildGroupCtx; + typedef struct SFilterInfo { + bool scalarMode; + SFltScalarCtx sclCtx; uint32_t options; uint32_t status; - uint16_t unitSize; - uint16_t unitNum; - uint16_t groupNum; - uint16_t colRangeNum; + uint32_t unitSize; + uint32_t unitNum; + uint32_t groupNum; + uint32_t colRangeNum; SFilterFields fields[FLD_TYPE_MAX]; SFilterGroup *groups; - uint16_t *cgroups; SFilterUnit *units; SFilterComUnit *cunits; uint8_t *unitRes; // result @@ -236,16 +253,15 @@ typedef struct SFilterInfo { SFilterRangeCtx **colRange; filter_exec_func func; uint8_t blkFlag; - uint16_t blkGroupNum; - uint16_t *blkUnits; + uint32_t blkGroupNum; + uint32_t *blkUnits; int8_t *blkUnitRes; - + void *pTable; + SFilterPCtx pctx; } SFilterInfo; -#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) - -#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) #define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) @@ -261,7 +277,7 @@ typedef struct SFilterInfo { #define FILTER_CLR_FLAG(st, f) st &= (~f) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) -#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) +#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0) #define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) #define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) @@ -271,15 +287,20 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) -#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) -#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) -#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) -#define CHK_RETV(c) do { if (c) { return; } } while (0) -#define CHK_RET(c, r) do { if (c) { return r; } } while (0) -#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) -#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) -#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) +#define fltFatal(...) qFatal(__VA_ARGS__) +#define fltError(...) qError(__VA_ARGS__) +#define fltWarn(...) qWarn(__VA_ARGS__) +#define fltInfo(...) qInfo(__VA_ARGS__) +#define fltDebug(...) qDebug(__VA_ARGS__) +#define fltTrace(...) qTrace(__VA_ARGS__) + + +#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0) +#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) @@ -290,6 +311,7 @@ typedef struct SFilterInfo { #define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) +#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) @@ -302,6 +324,7 @@ typedef struct SFilterInfo { #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) +#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) #define FILTER_UNIT_OPTR(u) ((u)->compare.optr) #define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) @@ -316,7 +339,7 @@ typedef struct SFilterInfo { #define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) #define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) +#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0) #define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) @@ -325,9 +348,10 @@ typedef struct SFilterInfo { #define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) #if 0 -extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); +extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); +extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp); +extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); @@ -344,4 +368,4 @@ extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); } #endif -#endif // TDENGINE_QFILTER_H +#endif // TDENGINE_FILTER_INT_H \ No newline at end of file diff --git a/source/libs/scalar/inc/tfilter.h b/source/libs/scalar/inc/tfilter.h deleted file mode 100644 index 27665da0c8..0000000000 --- a/source/libs/scalar/inc/tfilter.h +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_QFILTER_H -#define TDENGINE_QFILTER_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "thash.h" -#include "tname.h" -#include "common.h" -#include "scalar.h" -#include "querynodes.h" - -#define FILTER_DEFAULT_GROUP_SIZE 4 -#define FILTER_DEFAULT_UNIT_SIZE 4 -#define FILTER_DEFAULT_FIELD_SIZE 4 -#define FILTER_DEFAULT_VALUE_SIZE 4 -#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2 - -#define FILTER_DUMMY_EMPTY_OPTR 127 - -#define MAX_NUM_STR_SIZE 40 - -#define FILTER_RM_UNIT_MIN_ROWS 100 - -enum { - FLD_TYPE_COLUMN = 1, - FLD_TYPE_VALUE = 2, - FLD_TYPE_MAX = 3, - FLD_DESC_NO_FREE = 4, - FLD_DATA_NO_FREE = 8, - FLD_DATA_IS_HASH = 16, -}; - -enum { - MR_ST_START = 1, - MR_ST_FIN = 2, - MR_ST_ALL = 4, - MR_ST_EMPTY = 8, -}; - -enum { - RANGE_FLG_EXCLUDE = 1, - RANGE_FLG_INCLUDE = 2, - RANGE_FLG_NULL = 4, -}; - -enum { - FI_OPTION_NO_REWRITE = 1, - FI_OPTION_TIMESTAMP = 2, - FI_OPTION_NEED_UNIQE = 4, -}; - -enum { - FI_STATUS_ALL = 1, - FI_STATUS_EMPTY = 2, - FI_STATUS_REWRITE = 4, - FI_STATUS_CLONED = 8, -}; - -enum { - FI_STATUS_BLK_ALL = 1, - FI_STATUS_BLK_EMPTY = 2, - FI_STATUS_BLK_ACTIVE = 4, -}; - -enum { - RANGE_TYPE_UNIT = 1, - RANGE_TYPE_VAR_HASH = 2, - RANGE_TYPE_MR_CTX = 3, -}; - -typedef struct OptrStr { - uint16_t optr; - char *str; -} OptrStr; - -typedef struct SFilterRange { - int64_t s; - int64_t e; - char sflag; - char eflag; -} SFilterRange; - -typedef struct SFilterColRange { - uint16_t idx; //column field idx - bool isNull; - bool notNull; - bool isRange; - SFilterRange ra; -} SFilterColRange; - -typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); -typedef int32_t(*filter_desc_compare_func)(const void *, const void *); -typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); - -typedef struct SFilterRangeCompare { - int64_t s; - int64_t e; - rangeCompFunc func; -} SFilterRangeCompare; - -typedef struct SFilterRangeNode { - struct SFilterRangeNode* prev; - struct SFilterRangeNode* next; - union { - SFilterRange ra; - SFilterRangeCompare rc; - }; -} SFilterRangeNode; - -typedef struct SFilterRangeCtx { - int32_t type; - int32_t options; - int8_t status; - bool isnull; - bool notnull; - bool isrange; - int16_t colId; - __compar_fn_t pCompareFunc; - SFilterRangeNode *rf; //freed - SFilterRangeNode *rs; -} SFilterRangeCtx ; - -typedef struct SFilterVarCtx { - int32_t type; - int32_t options; - int8_t status; - bool isnull; - bool notnull; - bool isrange; - SHashObj *wild; - SHashObj *value; -} SFilterVarCtx; - -typedef struct SFilterField { - uint16_t flag; - void* desc; - void* data; -} SFilterField; - -typedef struct SFilterFields { - uint16_t size; - uint16_t num; - SFilterField *fields; -} SFilterFields; - -typedef struct SFilterFieldId { - uint16_t type; - uint16_t idx; -} SFilterFieldId; - -typedef struct SFilterGroup { - uint16_t unitSize; - uint16_t unitNum; - uint16_t *unitIdxs; - uint8_t *unitFlags; // !unit result -} SFilterGroup; - -typedef struct SFilterColInfo { - uint8_t type; - int32_t dataType; - void *info; -} SFilterColInfo; - -typedef struct SFilterGroupCtx { - uint16_t colNum; - uint16_t *colIdx; - SFilterColInfo *colInfo; -} SFilterGroupCtx; - -typedef struct SFilterColCtx { - uint16_t colIdx; - void* ctx; -} SFilterColCtx; - -typedef struct SFilterCompare { - uint8_t type; - uint8_t optr; - uint8_t optr2; -} SFilterCompare; - -typedef struct SFilterUnit { - SFilterCompare compare; - SFilterFieldId left; - SFilterFieldId right; - SFilterFieldId right2; -} SFilterUnit; - -typedef struct SFilterComUnit { - void *colData; - void *valData; - void *valData2; - uint16_t colId; - uint16_t dataSize; - uint8_t dataType; - uint8_t optr; - int8_t func; - int8_t rfunc; -} SFilterComUnit; - -typedef struct SFilterPCtx { - SHashObj *valHash; - SHashObj *unitHash; -} SFilterPCtx; - -typedef struct SFilterInfo { - uint32_t options; - uint32_t status; - uint16_t unitSize; - uint16_t unitNum; - uint16_t groupNum; - uint16_t colRangeNum; - SFilterFields fields[FLD_TYPE_MAX]; - SFilterGroup *groups; - uint16_t *cgroups; - SFilterUnit *units; - SFilterComUnit *cunits; - uint8_t *unitRes; // result - uint8_t *unitFlags; // got result - SFilterRangeCtx **colRange; - filter_exec_func func; - uint8_t blkFlag; - uint16_t blkGroupNum; - uint16_t *blkUnits; - int8_t *blkUnitRes; - - SFilterPCtx pctx; -} SFilterInfo; - -#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) - -#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR) -#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) - -#define MR_EMPTY_RES(ctx) (ctx->rs == NULL) - -#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0) -#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0) -#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) -#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) - - -#define FILTER_GET_FLAG(st, f) (st & f) -#define FILTER_SET_FLAG(st, f) st |= (f) -#define FILTER_CLR_FLAG(st, f) st &= (~f) - -#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) -#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0) -#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) -#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) - -#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0) -#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0) -#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0) -#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) -#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) - -#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) -#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) -#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) - -#define CHK_RETV(c) do { if (c) { return; } } while (0) -#define CHK_RET(c, r) do { if (c) { return r; } } while (0) -#define CHK_JMP(c) do { if (c) { goto _return; } } while (0) -#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0) -#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0) - -#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) -#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) -#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) -#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) -#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) - -#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid]) -#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left) -#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right) -#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2) -#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type) -#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u)) -#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri) -#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u)) -#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u)) -#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u)) -#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx) -#define FILTER_UNIT_OPTR(u) ((u)->compare.optr) -#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func) - -#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags)) -#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1 -#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx]) -#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx]) -#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v) - -#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0) -#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0) -#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0) - -#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0) - -#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0) - - -#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL) -#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY) - -#if 0 -extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); -extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); -extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); -extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); -extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); -#else -//REMOVE THESE!!!!!!!!!!!!!!!!!!!! -#include "function.h" -#endif -extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); -extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_QFILTER_H diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index b680ac6778..cf84e45d4c 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -17,7 +17,7 @@ #include "thash.h" //#include "queryLog.h" #include "tcompare.h" -#include "filter.h" +#include "filterInt.h" OptrStr gOptrStr[] = { {TSDB_RELATION_INVALID, "invalid"}, @@ -28,21 +28,22 @@ OptrStr gOptrStr[] = { {TSDB_RELATION_GREATER_EQUAL, ">="}, {TSDB_RELATION_NOT_EQUAL, "!="}, {TSDB_RELATION_LIKE, "like"}, - {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_MATCH, "nmatch"}, {TSDB_RELATION_ISNULL, "is null"}, {TSDB_RELATION_NOTNULL, "not null"}, {TSDB_RELATION_IN, "in"}, {TSDB_RELATION_AND, "and"}, {TSDB_RELATION_OR, "or"}, - {TSDB_RELATION_NOT, "not"} + {TSDB_RELATION_NOT, "not"}, + {TSDB_RELATION_MATCH, "match"}, + {TSDB_RELATION_NMATCH, "nmatch"}, + {TSDB_RELATION_CONTAINS, "contains"}, }; static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { const SSchema *sch1 = desc1; const SSchema *sch2 = desc2; - return sch1->colId != sch2->colId; + return !(strcmp(sch1->name, sch2->name) == 0 && sch1->colId == sch2->colId); } static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { @@ -60,16 +61,24 @@ filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) >= 0; + int32_t result = cfunc(maxv, minr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(maxv, minr) > 0; + int32_t result = cfunc(maxv, minr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) <= 0; + int32_t result = cfunc(minv, maxr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { - return cfunc(minv, maxr) < 0; + int32_t result = cfunc(minv, maxr); + if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; @@ -291,7 +300,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) { info->fields[FLD_TYPE_COLUMN].num = 0; info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; - info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); + info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField)); info->fields[FLD_TYPE_VALUE].num = 0; info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); @@ -318,7 +327,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt void* filterInitRangeCtx(int32_t type, int32_t options) { if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - //qError("not supported range type:%d", type); + qError("not supported range type:%d", type); return NULL; } @@ -685,7 +694,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) { } if (num == 0) { - //qError("no range result"); + qError("no range result"); return TSDB_CODE_QRY_APP_ERROR; } @@ -775,8 +784,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { int32_t leftSize = (int32_t)taosArrayGetSize(left); int32_t rightSize = (int32_t)taosArrayGetSize(right); -// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); -// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + if (taosArrayGetSize(left) <= 0) { + fltDebug("empty group"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (taosArrayGetSize(right) <= 0) { + fltDebug("empty group"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } for (int32_t l = 0; l < leftSize; ++l) { SFilterGroup *gp1 = taosArrayGet(left, l); @@ -793,7 +809,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { } int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { - for (uint16_t i = 0; i < fields->num; ++i) { + for (uint32_t i = 0; i < fields->num; ++i) { if (0 == gDescCompare[type](fields->fields[i].desc, v)) { return i; } @@ -806,7 +822,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { if (type == FLD_TYPE_VALUE) { if (info->pctx.valHash == false) { - //qError("value hash is empty"); + qError("value hash is empty"); return -1; } @@ -819,10 +835,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d return -1; } - +// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0 +// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { int32_t idx = -1; - uint16_t *num; + uint32_t *num; num = &info->fields[type].num; @@ -885,8 +902,14 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { -// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); -// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR); + if (node == NULL) { + fltError("empty node"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE) { + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } int32_t type; void *v; @@ -906,7 +929,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI return TSDB_CODE_SUCCESS; } -int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { if (info->pctx.unitHash == NULL) { info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); @@ -915,14 +938,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); if (hu) { - *uidx = *(uint16_t *)hu; + *uidx = *(uint32_t *)hu; return TSDB_CODE_SUCCESS; } } } if (info->unitNum >= info->unitSize) { - uint16_t psize = info->unitSize; + uint32_t psize = info->unitSize; info->unitSize += FILTER_DEFAULT_UNIT_SIZE; info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); @@ -940,7 +963,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { - assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); + if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){ + return -1; + } } SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); @@ -963,7 +988,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi -int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { +int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { group->unitSize += FILTER_DEFAULT_UNIT_SIZE; group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); @@ -974,12 +999,14 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { return TSDB_CODE_SUCCESS; } -int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { +int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType, bool tolower) { SBufferReader br = tbufInitReader(buf, len, false); uint32_t sType = tbufReadUint32(&br); SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); int32_t code = 0; - + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); + int dummy = -1; SVariant tmpVar = {0}; size_t t = 0; @@ -1065,47 +1092,53 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3 switch (tType) { case TSDB_DATA_TYPE_BOOL: + if (sType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(sType)) { + goto _return; + } + if (tmpVar.i64 > 1 ||tmpVar.i64 < 0) { + goto _return; + } case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } pvar = &val; t = sizeof(val); break; } case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } pvar = &val; t = sizeof(val); break; } case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: { -// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } -// -// goto _return; -// } + if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } + + goto _return; + } pvar = &val; t = sizeof(val); break; @@ -1129,15 +1162,15 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3 break; } case TSDB_DATA_TYPE_FLOAT: { -// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { -// if (converted) { -// taosVariantDestroy(&tmpVar); -// memset(&tmpVar, 0, sizeof(tmpVar)); -// continue; -// } + if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { + if (converted) { + tVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + continue; + } -// goto _return; -// } + goto _return; + } pvar = &val; t = sizeof(val); break; @@ -1148,6 +1181,10 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3 } t = varDataLen(tmp); pvar = varDataVal(tmp); + + if (tolower) { + strntolower_s(pvar, pvar, (int32_t)t); + } break; } case TSDB_DATA_TYPE_NCHAR: { @@ -1178,22 +1215,51 @@ _return: return code; } +static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pLeft) { + if((*pLeft)->nodeType == TSQL_NODE_EXPR && (*pLeft)->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation + assert(info->pTable != NULL); + SSchema* schema = (*pLeft)->_node.pLeft->pSchema; + if((*pLeft)->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5); + memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); + (*pLeft) = (*pLeft)->_node.pLeft; // -> operation use left as input + }else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) && + (tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){ + SSchema* schema = (*pLeft)->pSchema; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + uint32_t nullData = TSDB_DATA_JSON_NULL; + jsonKeyMd5(&nullData, INT_BYTES, keyMd5); + memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); + }else if(tree->_node.optr == TSDB_RELATION_CONTAINS){ + SSchema* schema = (*pLeft)->pSchema; + if(tree->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5); + memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); + } + return TSDB_CODE_SUCCESS; +} -int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { +int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { + tExprNode* pLeft = tree->_node.pLeft; + int32_t ret = TSDB_CODE_SUCCESS; + if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret; SFilterFieldId left = {0}, right = {0}; - - filterAddFieldFromNode(info, tree->_node.pLeft, &left); - - SVariant* var = tree->_node.pRight->pVal; - int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); - size_t len = 0; - uint16_t uidx = 0; + filterAddFieldFromNode(info, pLeft, &left); + uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + int32_t len = 0; + uint32_t uidx = 0; if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { void *data = NULL; - filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); -// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + SVariant* var = tree->_node.pRight->pVal; + filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false); + if (data == NULL) { + fltError("failed to convert in param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } if (taosHashGetSize((SHashObj *)data) <= 0) { filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); @@ -1209,23 +1275,13 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g void *p = taosHashIterate((SHashObj *)data, NULL); while(p) { - void* key = NULL; - len = 0; - - taosHashGetKey(p, &key, &len); + void *key = taosHashGetDataKey((SHashObj *)data, p); void *fdata = NULL; - - if (IS_VAR_DATA_TYPE(type)) { - fdata = malloc(len + VARSTR_HEADER_SIZE); - varDataLen(fdata) = len; - memcpy(varDataVal(fdata), key, len); - len += VARSTR_HEADER_SIZE; - } else { - fdata = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(fdata, key); - len = tDataTypes[type].bytes; - } - + + fdata = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(fdata, key); + len = tDataTypes[type].bytes; + filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); @@ -1240,9 +1296,10 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g taosHashCleanup(data); } else { - filterAddFieldFromNode(info, tree->_node.pRight, &right); + filterAddFieldFromNode(info, tree->_node.pRight, &right); - filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + ret = filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + CHK_LRET(ret != TSDB_CODE_SUCCESS, TSDB_CODE_QRY_APP_ERROR, "invalid where condition"); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); @@ -1254,7 +1311,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g } -int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { +int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; int32_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = FLD_DESC_NO_FREE; @@ -1267,7 +1324,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u void *data = FILTER_UNIT_VAL_DATA(src, u); if (IS_VAR_DATA_TYPE(type)) { if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); @@ -1289,7 +1346,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); } -int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { +int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) { SFilterUnit *u = &info->units[uidx]; u->compare.optr2 = optr; @@ -1299,9 +1356,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ } -int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { +int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { SFilterFieldId left, right, right2; - uint16_t uidx = 0; + uint32_t uidx = 0; SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); @@ -1479,61 +1536,80 @@ static void filterFreeGroup(void *pItem) { } -int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { +EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { int32_t code = TSDB_CODE_SUCCESS; - SArray* leftGroup = NULL; - SArray* rightGroup = NULL; - - if (tree->nodeType != TEXPR_BINARYEXPR_NODE) { - //qError("invalid nodeType:%d", tree->nodeType); - return TSDB_CODE_QRY_APP_ERROR; + SArray* preGroup = NULL; + SArray* newGroup = NULL; + SArray* resGroup = NULL; + ENodeType nType = nodeType(nType); + SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext; + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { + SLogicConditionNode *node = (SLogicConditionNode *)pNode; + if (LOGIC_COND_TYPE_AND == node->condType) { + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + newGroup = taosArrayInit(4, sizeof(SFilterGroup)); + resGroup = taosArrayInit(4, sizeof(SFilterGroup)); + + SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup}; + nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx); + FLT_ERR_JRET(tctx.code); + + FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup)); + + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + + preGroup = resGroup; + } + + taosArrayAddAll(ctx->group, resGroup); + + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + taosArrayDestroyEx(resGroup, filterFreeGroup); + + return DEAL_RES_IGNORE_CHILD; + } + + if (LOGIC_COND_TYPE_OR == node->condType) { + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); + FLT_ERR_JRET(ctx.code); + } + + return DEAL_RES_IGNORE_CHILD; + } + + fltError("invalid condition type, type:%d", node->condType); + + return DEAL_RES_ERROR; } - - if (tree->_node.optr == TSDB_RELATION_AND) { - leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); - rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); - ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); - ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); - - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); + if (QUERY_NODE_OPERATOR == nType) { + FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group)); - return TSDB_CODE_SUCCESS; + return DEAL_RES_CONTINUE; } - if (tree->_node.optr == TSDB_RELATION_OR) { - ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); - ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); - - return TSDB_CODE_SUCCESS; - } - - code = filterAddGroupUnitFromNode(info, tree, group); + sclError("invalid node type for filter, type:%d", nodeType(pNode)); + code = TSDB_CODE_QRY_INVALID_INPUT; _return: - taosArrayDestroyEx(leftGroup, filterFreeGroup); - taosArrayDestroyEx(rightGroup, filterFreeGroup); - - return code; + taosArrayDestroyEx(newGroup, filterFreeGroup); + taosArrayDestroyEx(preGroup, filterFreeGroup); + taosArrayDestroyEx(resGroup, filterFreeGroup); + + ctx->code = code; + + return DEAL_RES_ERROR; } -#if 0 -int32_t filterInitUnitFunc(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { - SFilterUnit* unit = &info->units[i]; - - info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); - } - - return TSDB_CODE_SUCCESS; -} -#endif - -int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { +int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { int32_t n = 0; switch (type) { @@ -1611,34 +1687,37 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { if (qDebugFlag & DEBUG_DEBUG) { -// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); + if (info == NULL) { + fltDebug("%s - FilterInfo: EMPTY", msg); + return; + } if (options == 0) { -// //qDebug("%s - FilterInfo:", msg); -// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + qDebug("%s - FilterInfo:", msg); + qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; SSchema *sch = field->desc; -// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); } - //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); - for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { + qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); + for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { SVariant *var = field->desc; if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { - //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); + qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); } else { - //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO } } else if (field->data) { - //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO } } - //qDebug("UNIT Num:%u", info->unitNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { + qDebug("UNIT Num:%u", info->unitNum); + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; int32_t type = FILTER_UNIT_DATA_TYPE(unit); int32_t len = 0; @@ -1647,7 +1726,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SSchema *sch = left->desc; - len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){ + len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + } if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); @@ -1656,7 +1737,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); + if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); } else { strcat(str, "NULL"); } @@ -1664,7 +1745,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_CONTAINS){ + sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + } if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); @@ -1673,23 +1756,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } - converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); } else { strcat(str, "NULL"); } strcat(str, "]"); } - //qDebug("%s", str); //TODO + qDebug("%s", str); //TODO } - //qDebug("GROUP Num:%u", info->groupNum); - for (uint16_t i = 0; i < info->groupNum; ++i) { + qDebug("GROUP Num:%u", info->groupNum); + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup *group = &info->groups[i]; - //qDebug("Group%d : unit num[%u]", i, group->unitNum); + qDebug("Group%d : unit num[%u]", i, group->unitNum); - for (uint16_t u = 0; u < group->unitNum; ++u) { - //qDebug("unit id:%u", group->unitIdxs[u]); + for (uint32_t u = 0; u < group->unitNum; ++u) { + qDebug("unit id:%u", group->unitIdxs[u]); } } @@ -1697,12 +1780,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) } if (options == 1) { - //qDebug("%s - RANGE info:", msg); + qDebug("%s - RANGE info:", msg); - //qDebug("RANGE Num:%u", info->colRangeNum); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + qDebug("RANGE Num:%u", info->colRangeNum); + for (uint32_t i = 0; i < info->colRangeNum; ++i) { SFilterRangeCtx *ctx = info->colRange[i]; - //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); if (ctx->isrange) { SFilterRangeNode *r = ctx->rs; while (r) { @@ -1712,7 +1795,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str,"(NULL)"); } else { FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); } strcat(str, " - "); @@ -1720,10 +1803,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) strcat(str, "(NULL)"); } else { FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); - converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); } - //qDebug("range: %s", str); + qDebug("range: %s", str); r = r->next; } @@ -1733,25 +1816,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) return; } - //qDebug("%s - Block Filter info:", msg); + qDebug("%s - Block Filter info:", msg); if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { - //qDebug("Flag:%s", "ALL"); + qDebug("Flag:%s", "ALL"); return; } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { - //qDebug("Flag:%s", "EMPTY"); + qDebug("Flag:%s", "EMPTY"); return; } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ - //qDebug("Flag:%s", "ACTIVE"); + qDebug("Flag:%s", "ACTIVE"); } - //qDebug("GroupNum:%d", info->blkGroupNum); - uint16_t *unitIdx = info->blkUnits; - for (uint16_t i = 0; i < info->blkGroupNum; ++i) { - //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); - uint16_t unitNum = *(unitIdx++); - for (uint16_t m = 0; m < unitNum; ++m) { - //qDebug("uidx[%d]", *(unitIdx++)); + qDebug("GroupNum:%d", info->blkGroupNum); + uint32_t *unitIdx = info->blkUnits; + for (uint32_t i = 0; i < info->blkGroupNum; ++i) { + qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); + uint32_t unitNum = *(unitIdx++); + for (uint32_t m = 0; m < unitNum; ++m) { + qDebug("uidx[%d]", *(unitIdx++)); } } } @@ -1837,21 +1920,23 @@ void filterFreePCtx(SFilterPCtx *pctx) { } void filterFreeInfo(SFilterInfo *info) { - CHK_RETV(info == NULL); + if (info == NULL) { + return; + } tfree(info->cunits); tfree(info->blkUnitRes); tfree(info->blkUnits); for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { - for (uint16_t f = 0; f < info->fields[i].num; ++f) { + for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); } tfree(info->fields[i].fields); } - for (int32_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { filterFreeGroup(&info->groups[i]); } @@ -1863,7 +1948,7 @@ void filterFreeInfo(SFilterInfo *info) { tfree(info->unitFlags); - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + for (uint32_t i = 0; i < info->colRangeNum; ++i) { filterFreeRangeCtx(info->colRange[i]); } @@ -1901,7 +1986,7 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { int32_t filterInitValFieldData(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); @@ -1923,8 +2008,11 @@ int32_t filterInitValFieldData(SFilterInfo *info) { } if (unit->compare.optr == TSDB_RELATION_IN) { - filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); -// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type, false); + if (fi->data == NULL) { + fltError("failed to convert in param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); @@ -1937,7 +2025,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) { } else if (type == TSDB_DATA_TYPE_NCHAR) { size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - } else { + } else if (type != TSDB_DATA_TYPE_JSON){ if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE fi->data = calloc(var->nLen, tDataTypes[type].bytes); for (int32_t a = 0; a < var->nLen; ++a) { @@ -1949,19 +2037,47 @@ int32_t filterInitValFieldData(SFilterInfo *info) { } else { fi->data = calloc(1, sizeof(int64_t)); } + } else{ // type == TSDB_DATA_TYPE_JSON + // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc) + } + + if(type != TSDB_DATA_TYPE_JSON){ + bool converted = false; + char extInfo = 0; + if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { + if (converted) { + filterHandleValueExtInfo(unit, extInfo); + + continue; + } + qError("dump value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + // match/nmatch for nchar type need convert from ucs4 to mbs + if(type == TSDB_DATA_TYPE_NCHAR && + (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); + if (len < 0){ + qError("filterInitValFieldData taosUcs4ToMbs error 1"); + return TSDB_CODE_FAILED; + } + varDataSetLen(newValData, len); + varDataCopy(fi->data, newValData); + }else if(type == TSDB_DATA_TYPE_JSON && + (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData); + if (len < 0){ + qError("filterInitValFieldData taosUcs4ToMbs error 2"); + return TSDB_CODE_FAILED; + } + memcpy(((tVariant*)(fi->desc))->pz, newValData, len); + ((tVariant*)(fi->desc))->nLen = len; } - bool converted = false; - char extInfo = 0; -// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { -// if (converted) { -// filterHandleValueExtInfo(unit, extInfo); -// -// continue; -// } -// //qError("dump value to type[%d] failed", type); -// return TSDB_CODE_TSC_INVALID_OPERATION; -// } } return TSDB_CODE_SUCCESS; @@ -1971,6 +2087,8 @@ int32_t filterInitValFieldData(SFilterInfo *info) { bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { int32_t ret = func(left, right); + if(ret == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + switch (optr) { case TSDB_RELATION_EQUAL: { return ret == 0; @@ -1996,6 +2114,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) case TSDB_RELATION_MATCH: { return ret == 0; } + case TSDB_RELATION_CONTAINS: { + return ret == 0; + } case TSDB_RELATION_NMATCH: { return ret == 0; } @@ -2092,7 +2213,7 @@ _return: } -int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { +int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) { SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; int32_t size = (int32_t)taosArrayGetSize(colArray); int32_t type = gRes->colInfo[colIdx].dataType; @@ -2109,6 +2230,10 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); CHK_JMP(MR_EMPTY_RES(ctx)); } + if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ + gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL; + SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u)); + } } taosArrayDestroy(colArray); @@ -2129,11 +2254,11 @@ _return: int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { bool empty = false; - uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); - uint16_t colIdxi = 0; - uint16_t gResIdx = 0; + uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t)); + uint32_t colIdxi = 0; + uint32_t gResIdx = 0; - for (uint16_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup* g = info->groups + i; gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); @@ -2141,9 +2266,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t colIdxi = 0; empty = false; - for (uint16_t j = 0; j < g->unitNum; ++j) { + for (uint32_t j = 0; j < g->unitNum; ++j) { SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); - uint16_t cidx = FILTER_UNIT_COL_IDX(u); + uint32_t cidx = FILTER_UNIT_COL_IDX(u); if (gRes[gResIdx]->colInfo[cidx].info == NULL) { gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); @@ -2159,10 +2284,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } if (colIdxi > 1) { - qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); } - for (uint16_t l = 0; l < colIdxi; ++l) { + for (uint32_t l = 0; l < colIdxi; ++l) { int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; if (FILTER_NO_MERGE_DATA_TYPE(type)) { @@ -2201,7 +2326,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t } void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool equal = false; for (; m < gRes1->colNum; ++m) { @@ -2224,6 +2349,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict = true; return; } + + // for long in operation + if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) { + SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info; + if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){ + *conflict = true; + return; + } + } ++n; equal = true; @@ -2241,7 +2375,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool } -int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { +int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); @@ -2271,10 +2405,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); - uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; bool equal = false; - uint16_t equal1 = 0, equal2 = 0, merNum = 0; + uint32_t equal1 = 0, equal2 = 0, merNum = 0; SFilterRangeCtx *ctx = NULL; SFilterColCtx colCtx = {0}; SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); @@ -2388,7 +2522,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); int32_t pEnd = 0, cStart = 0, cEnd = 0; - uint16_t pColNum = 0, cColNum = 0; + uint32_t pColNum = 0, cColNum = 0; int32_t movedNum = 0; bool all = false; @@ -2454,7 +2588,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR } cStart = i; - cEnd = i; cColNum = gRes[i]->colNum; } @@ -2470,7 +2603,7 @@ _return: int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { size_t groupSize = taosArrayGetSize(group); - info->groupNum = (uint16_t)groupSize; + info->groupNum = (uint32_t)groupSize; if (info->groupNum > 0) { info->groups = calloc(info->groupNum, sizeof(*info->groups)); @@ -2487,7 +2620,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { - //qDebug("no need rewrite"); + qDebug("no need rewrite"); return TSDB_CODE_SUCCESS; } @@ -2499,7 +2632,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum SFilterGroupCtx *res = NULL; SFilterColInfo *colInfo = NULL; int32_t optr = 0; - uint16_t uidx = 0; + uint32_t uidx = 0; memset(info, 0, sizeof(*info)); @@ -2519,7 +2652,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum SFilterGroup ng = {0}; - for (uint16_t m = 0; m < res->colNum; ++m) { + for (uint32_t m = 0; m < res->colNum; ++m) { colInfo = &res->colInfo[res->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { assert(colInfo->type == RANGE_TYPE_UNIT); @@ -2555,13 +2688,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum } int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { - uint16_t *idxs = NULL; - uint16_t colNum = 0; + uint32_t *idxs = NULL; + uint32_t colNum = 0; SFilterGroupCtx *res = NULL; - uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); for (int32_t i = 0; i < gResNum; ++i) { - for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { + for (uint32_t m = 0; m < gRes[i]->colNum; ++m) { SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { continue; @@ -2571,7 +2704,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ } } - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { if (idxNum[i] < gResNum) { continue; } @@ -2592,9 +2725,9 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ for (int32_t i = 0; i < gResNum; ++i) { res = gRes[i]; - uint16_t n = 0; + uint32_t n = 0; - for (uint16_t m = 0; m < info->colRangeNum; ++m) { + for (uint32_t m = 0; m < info->colRangeNum; ++m) { for (; n < res->colNum; ++n) { if (res->colIdx[n] < idxs[m]) { continue; @@ -2642,7 +2775,7 @@ _return: } int32_t filterPostProcessRange(SFilterInfo *info) { - for (uint16_t i = 0; i < info->colRangeNum; ++i) { + for (uint32_t i = 0; i < info->colRangeNum; ++i) { SFilterRangeCtx* ctx = info->colRange[i]; SFilterRangeNode *r = ctx->rs; while (r) { @@ -2656,13 +2789,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) { int32_t filterGenerateComInfo(SFilterInfo *info) { - uint16_t n = 0; - info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); @@ -2672,7 +2803,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); if (unit->right.type == FLD_TYPE_VALUE) { - info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant + info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit); + }else{ + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } } else { info->cunits[i].valData = NULL; } @@ -2685,30 +2820,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) { info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); } - - uint16_t cgroupNum = info->groupNum + 1; - - for (uint16_t i = 0; i < info->groupNum; ++i) { - cgroupNum += info->groups[i].unitNum; - } - - info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); - - for (uint16_t i = 0; i < info->groupNum; ++i) { - info->cgroups[n++] = info->groups[i].unitNum; - - for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { - info->cgroups[n++] = info->groups[i].unitIdxs[m]; - } - } - - info->cgroups[n] = 0; return TSDB_CODE_SUCCESS; } int32_t filterUpdateComUnits(SFilterInfo *info) { - for (uint16_t i = 0; i < info->unitNum; ++i) { + for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); @@ -2723,7 +2840,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); - for (int32_t k = 0; k < info->unitNum; ++k) { + for (uint32_t k = 0; k < info->unitNum; ++k) { int32_t index = -1; SFilterComUnit *cunit = &info->cunits[k]; @@ -2768,18 +2885,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } } - if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL + if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL) { + || cunit->optr == TSDB_RELATION_NOT_EQUAL || cunit->optr == TSDB_RELATION_CONTAINS) { continue; } SColumnDataAgg* pDataBlockst = &pDataStatis[index]; void *minVal, *maxVal; + float minv = 0; + float maxv = 0; if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); + minv = (float)(*(double *)(&pDataBlockst->min)); + maxv = (float)(*(double *)(&pDataBlockst->max)); minVal = &minv; maxVal = &maxv; @@ -2834,12 +2953,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } -// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); + if (rmUnit == 0) { + fltDebug("NO Block Filter APPLY"); + FLT_RET(TSDB_CODE_SUCCESS); + } info->blkGroupNum = info->groupNum; - uint16_t *unitNum = info->blkUnits; - uint16_t *unitIdx = unitNum + 1; + uint32_t *unitNum = info->blkUnits; + uint32_t *unitIdx = unitNum + 1; int32_t all = 0, empty = 0; for (uint32_t g = 0; g < info->groupNum; ++g) { @@ -2849,7 +2971,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 empty = 0; for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + uint32_t uidx = group->unitIdxs[u]; if (info->blkUnitRes[uidx] == 1) { --(*unitNum); all = 1; @@ -2896,17 +3018,19 @@ _return: bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; - uint16_t *unitIdx = NULL; - - *p = calloc(numOfRows, sizeof(int8_t)); + uint32_t *unitIdx = NULL; + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); unitIdx = info->blkUnits; for (uint32_t g = 0; g < info->blkGroupNum; ++g) { - uint16_t unitNum = *(unitIdx++); + uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; void *colData = (char *)cunit->colData + cunit->dataSize * i; @@ -3001,12 +3125,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, return all; } - *p = calloc(numOfRows, sizeof(int8_t)); + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if (!colData){ // for json->'key' is null + (*p)[i] = 1; + }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null + colData = POINTER_SHIFT(colData, CHAR_BYTES); + (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + }else{ + (*p)[i] = 0; + } + }else{ + (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType)); + } if ((*p)[i] == 0) { all = false; } @@ -3022,12 +3159,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } - *p = calloc(numOfRows, sizeof(int8_t)); - + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + if (!colData) { // for json->'key' is not null + (*p)[i] = 0; + }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null + colData = POINTER_SHIFT(colData, CHAR_BYTES); + (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + }else{ // for json->'key' is not null + (*p)[i] = 1; + } + }else { + (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType)); + } + if ((*p)[i] == 0) { all = false; } @@ -3036,7 +3188,48 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } -bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { +static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){ + if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){ + uint8_t jsonType = *(char*)colData; + char* realData = POINTER_SHIFT(colData, CHAR_BYTES); + if (jsonType != TSDB_DATA_TYPE_NCHAR){ + *result = false; + }else{ + char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + tfree(newColData); + return; + } + varDataSetLen(newColData, len); + tVariant* val = cunit->valData; + char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; + assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); + memcpy(varDataVal(newValData), val->pz, val->nLen); + varDataSetLen(newValData, val->nLen); + *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData); + tfree(newColData); + } + }else if(cunit->optr == TSDB_RELATION_LIKE){ + uint8_t jsonType = *(char*)colData; + char* realData = POINTER_SHIFT(colData, CHAR_BYTES); + if (jsonType != TSDB_DATA_TYPE_NCHAR){ + *result = false; + }else{ + tVariant* val = cunit->valData; + char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1); + memcpy(varDataVal(newValData), val->pz, val->nLen); + varDataSetLen(newValData, val->nLen); + *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData); + tfree(newValData); + } + }else{ + *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } +} + +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; @@ -3050,10 +3243,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD return all; } - *p = calloc(numOfRows, sizeof(int8_t)); + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - if (isNull(colData, info->cunits[0].dataType)) { + if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { all = false; colData += dataSize; continue; @@ -3078,19 +3273,37 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { return all; } - - *p = calloc(numOfRows, sizeof(int8_t)); + + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } for (int32_t i = 0; i < numOfRows; ++i) { - uint16_t uidx = info->groups[0].unitIdxs[0]; + uint32_t uidx = info->groups[0].unitIdxs[0]; void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - if (isNull(colData, info->cunits[uidx].dataType)) { + if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { + (*p)[i] = 0; all = false; continue; } + // match/nmatch for nchar type need convert from ucs4 to mbs + + if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){ + char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); + } + tfree(newColData); + }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ + doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData); + }else{ + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + } - (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); - if ((*p)[i] == 0) { all = false; } @@ -3108,15 +3321,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg return all; } - *p = calloc(numOfRows, sizeof(int8_t)); - + if (*p == NULL) { + *p = calloc(numOfRows, sizeof(int8_t)); + } + for (int32_t i = 0; i < numOfRows; ++i) { //FILTER_UNIT_CLR_F(info); for (uint32_t g = 0; g < info->groupNum; ++g) { SFilterGroup *group = &info->groups[g]; for (uint32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; void *colData = (char *)cunit->colData + cunit->dataSize * i; @@ -3125,7 +3340,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg //} else { uint8_t optr = cunit->optr; - if (isNull(colData, cunit->dataType)) { + if (colData == NULL || isNull(colData, cunit->dataType)) { (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; } else { if (optr == TSDB_RELATION_NOTNULL) { @@ -3135,7 +3350,21 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg } else if (cunit->rfunc >= 0) { (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); } else { - (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){ + char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); + int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData)); + if (len < 0){ + qError("castConvert1 taosUcs4ToMbs error"); + }else{ + varDataSetLen(newColData, len); + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); + } + tfree(newColData); + }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){ + doJsonCompare(cunit, &(*p)[i], colData); + }else{ + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } } //FILTER_UNIT_SET_R(info, uidx, p[i]); @@ -3211,13 +3440,13 @@ int32_t filterPreprocess(SFilterInfo *info) { filterMergeGroups(info, gRes, &gResNum); if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { -// qInfo("Final - FilterInfo: [ALL]"); + qInfo("Final - FilterInfo: [ALL]"); goto _return; } if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { -// qInfo("Final - FilterInfo: [EMPTY]"); + qInfo("Final - FilterInfo: [EMPTY]"); goto _return; } @@ -3244,9 +3473,29 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { -// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); -// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); +int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); + CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); + + if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema* sch = fi->desc; + + (*fp)(param, sch->colId, &fi->data); + } + + filterUpdateComUnits(info); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) { + CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); + CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { return TSDB_CODE_SUCCESS; @@ -3255,15 +3504,8 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; SSchema* sch = fi->desc; - - for (int32_t j = 0; j < numOfCols; ++j) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); - if (sch->colId == pColInfo->info.colId) { - fi->data = pColInfo->pData; - - break; - } - } + + (*fp)(param, sch->colId, sch->name, &fi->data); } filterUpdateComUnits(info); @@ -3271,46 +3513,32 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat return TSDB_CODE_SUCCESS; } - -int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { +int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { int32_t code = TSDB_CODE_SUCCESS; - SFilterInfo *info = NULL; - -// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); - - if (*pinfo == NULL) { - *pinfo = calloc(1, sizeof(SFilterInfo)); - } - - info = *pinfo; - - info->options = options; SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); filterInitUnitsFields(info); - code = filterTreeToGroup(tree, info, group); - - ERR_JRET(code); + SFltBuildGroupCtx tctx = {.info = info, .group = group}; + nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx); + FLT_ERR_JRET(tctx.code); filterConvertGroupFromArray(info, group); + taosArrayDestroy(group); - ERR_JRET(filterInitValFieldData(info)); + FLT_ERR_JRET(filterInitValFieldData(info)); if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { filterDumpInfoToString(info, "Before preprocess", 0); - ERR_JRET(filterPreprocess(info)); + FLT_ERR_JRET(filterPreprocess(info)); - CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); + FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - taosArrayDestroy(group); return code; } - - //ERR_JRET(filterInitUnitFunc(info)); } info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); @@ -3318,18 +3546,12 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option filterDumpInfoToString(info, "Final", 0); - taosArrayDestroy(group); - return code; _return: -// qInfo("No filter, code:%d", code); + + qInfo("No filter, code:%d", code); - taosArrayDestroy(group); - filterFreeInfo(*pinfo); - - *pinfo = NULL; - return code; } @@ -3348,7 +3570,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t bool ret = true; void *minVal, *maxVal; - for (int32_t k = 0; k < info->colRangeNum; ++k) { + for (uint32_t k = 0; k < info->colRangeNum; ++k) { int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for(int32_t i = 0; i < numOfCols; ++i) { @@ -3368,30 +3590,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t break; } - if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { - ret = false; - break; - } - - // all data in current column are NULL, no need to check its boundary value - if (pDataStatis[index].numOfNull == numOfRows) { - - // if isNULL query exists, load the null data column - if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + if (pDataStatis[index].numOfNull <= 0) { + if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } + } else if (pDataStatis[index].numOfNull > 0) { + if (pDataStatis[index].numOfNull == numOfRows) { + if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + ret = false; + break; + } - continue; + continue; + } else { + if (ctx->isnull) { + continue; + } + } } SColumnDataAgg* pDataBlockst = &pDataStatis[index]; SFilterRangeNode *r = ctx->rs; + float minv = 0; + float maxv = 0; if (ctx->type == TSDB_DATA_TYPE_FLOAT) { - float minv = (float)(*(double *)(&pDataBlockst->min)); - float maxv = (float)(*(double *)(&pDataBlockst->max)); + minv = (float)(*(double *)(&pDataBlockst->min)); + maxv = (float)(*(double *)(&pDataBlockst->max)); minVal = &minv; maxVal = &maxv; @@ -3408,7 +3635,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t r = r->next; } - CHK_RET(!ret, ret); + if (!ret) { + return ret; + } } return ret; @@ -3426,7 +3655,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { int32_t code = 0; bool empty = false, all = false; - for (int32_t i = 0; i < info->groupNum; ++i) { + for (uint32_t i = 0; i < info->groupNum; ++i) { SFilterGroup *group = &info->groups[i]; if (group->unitNum > 1) { cur = tmpc; @@ -3436,8 +3665,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { optr = TSDB_RELATION_OR; } - for (int32_t u = 0; u < group->unitNum; ++u) { - uint16_t uidx = group->unitIdxs[u]; + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint32_t uidx = group->unitIdxs[u]; SFilterUnit *unit = &info->units[uidx]; uint8_t raOptr = FILTER_UNIT_OPTR(unit); @@ -3478,8 +3707,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { } else { filterGetRangeNum(prev, &num); if (num > 1) { - //qError("only one time range accepted, num:%d", num); - ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); + qError("only one time range accepted, num:%d", num); + FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); } CHK_JMP(num < 1); @@ -3493,7 +3722,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { filterFreeRangeCtx(prev); filterFreeRangeCtx(tmpc); - //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); return TSDB_CODE_SUCCESS; _return: @@ -3503,14 +3732,18 @@ _return: filterFreeRangeCtx(prev); filterFreeRangeCtx(tmpc); - //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); return code; } int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { @@ -3523,7 +3756,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar char *src = FILTER_GET_COL_FIELD_DATA(fi, j); char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); int32_t len = 0; - taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + char *varSrc = varDataVal(src); + size_t k = 0, varSrcLen = varDataLen(src); + while (k < varSrcLen && varSrc[k++] == -1) {} + if (k == varSrcLen) { + /* NULL */ + varDataLen(dst) = (VarDataLenT) varSrcLen; + varDataCopy(dst, src); + continue; + } + bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + if(!ret) { + qError("filterConverNcharColumns taosMbsToUcs4 error"); + return TSDB_CODE_FAILED; + } varDataLen(dst) = len; } @@ -3541,7 +3787,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar } int32_t filterFreeNcharColumns(SFilterInfo* info) { - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); if (type == TSDB_DATA_TYPE_NCHAR) { @@ -3552,7 +3798,74 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { return TSDB_CODE_SUCCESS; } - +################################################################################## + +EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { + SFltTreeStat *stat = (SFltTreeStat *)pContext; +} + +int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat); + + FLT_RET(pStat->code); +} + +int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { + +} + + +int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) { + int32_t code = 0; + SFilterInfo *info = NULL; + + if (pNode == NULL || pInfo == NULL) { + fltError("invalid param"); + FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + if (*pInfo == NULL) { + *pInfo = calloc(1, sizeof(SFilterInfo)); + if (NULL == *pInfo) { + fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo)); + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + info = *pInfo; + info->options = options; + + SFltTreeStat stat = {0}; + FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + + if (!info->scalarMode) { + FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); + } + + FLT_ERR_JRET(fltOptimizeNodes(info, &pNode, &stat)); + + return code; + +_return: + + filterFreeInfo(*pInfo); + + *pInfo = NULL; + + FLT_RET(code); +} + +FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + if (info->scalarMode) { + SScalarParam output = {0}; + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); + + *p = output.data; + return TSDB_CODE_SUCCESS; + } + + return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); +} diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 082dd6cfd4..d13589545a 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -45,6 +45,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t break; } + case QUERY_NODE_NODE_LIST: { + SNodeListNode *nodeList = (SNodeListNode *)node; + //TODO BUILD HASH + break; + } case QUERY_NODE_COLUMN_REF: { if (NULL == ctx) { sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 07f32c87d8..57e9bc0d09 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -19,7 +19,7 @@ #include "sclvector.h" #include "tcompare.h" #include "querynodes.h" -#include "filter.h" +#include "filterInt.h" #include "query.h" //GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); From 46b7eb0dee221827e5b594d737c786e7efaa7c26 Mon Sep 17 00:00:00 2001 From: dapan Date: Sat, 19 Feb 2022 17:27:10 +0800 Subject: [PATCH 7/9] feature/qnode --- source/libs/scalar/inc/filterInt.h | 14 +- source/libs/scalar/src/filter.c | 519 +++++++++++------------------ 2 files changed, 195 insertions(+), 338 deletions(-) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index a23e698deb..ad40dc5958 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -223,6 +223,7 @@ typedef struct SFilterPCtx { typedef struct SFltTreeStat { int32_t code; + bool scalarMode; } SFltTreeStat; typedef struct SFltScalarCtx { @@ -287,6 +288,7 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) +#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL) #define fltFatal(...) qFatal(__VA_ARGS__) #define fltError(...) qError(__VA_ARGS__) @@ -304,12 +306,12 @@ typedef struct SFilterInfo { #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri)) -#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) +#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) #define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index cf84e45d4c..57c8e8c984 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -39,27 +39,6 @@ OptrStr gOptrStr[] = { {TSDB_RELATION_CONTAINS, "contains"}, }; -static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { - const SSchema *sch1 = desc1; - const SSchema *sch2 = desc2; - - return !(strcmp(sch1->name, sch2->name) == 0 && sch1->colId == sch2->colId); -} - -static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { - const SVariant *val1 = desc1; - const SVariant *val2 = desc2; - - return taosVariantCompare(val1, val2); -} - - -filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { - NULL, - filterFieldColDescCompare, - filterFieldValDescCompare -}; - bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; @@ -810,7 +789,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { for (uint32_t i = 0; i < fields->num; ++i) { - if (0 == gDescCompare[type](fields->fields[i].desc, v)) { + if (nodesEqualNode(fields->fields[i].desc, v)) { return i; } } @@ -901,27 +880,25 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte } -int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { +int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { if (node == NULL) { fltError("empty node"); FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE) { + if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) { FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t type; void *v; - if (node->nodeType == TEXPR_BINARYEXPR_NODE) { + if (nodeType(node) == QUERY_NODE_COLUMN_REF) { type = FLD_TYPE_COLUMN; - v = node->pSchema; - node->pSchema = NULL; + v = node; } else { type = FLD_TYPE_VALUE; - v = node->pVal; - node->pVal = NULL; + v = node; } filterAddField(info, v, NULL, type, fid, 0, true); @@ -999,287 +976,36 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { return TSDB_CODE_SUCCESS; } -int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType, bool tolower) { - SBufferReader br = tbufInitReader(buf, len, false); - uint32_t sType = tbufReadUint32(&br); - SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); - int32_t code = 0; - - taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType)); - - int dummy = -1; - SVariant tmpVar = {0}; - size_t t = 0; - int32_t sz = tbufReadInt32(&br); - void *pvar = NULL; - int64_t val = 0; - int32_t bufLen = 0; - if (IS_NUMERIC_TYPE(sType)) { - bufLen = 60; // The maximum length of string that a number is converted to. - } else { - bufLen = 128; - } - - char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); - - for (int32_t i = 0; i < sz; i++) { - switch (sType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)&val = tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(float *)&val = (float)tbufReadDouble(&br); - t = sizeof(val); - pvar = &val; - break; - } - case TSDB_DATA_TYPE_BINARY: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - case TSDB_DATA_TYPE_NCHAR: { - pvar = (char *)tbufReadBinary(&br, &t); - break; - } - default: - taosHashCleanup(pObj); - *q = NULL; - assert(0); - } - - taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); - - if (bufLen < t) { - tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); - bufLen = (int32_t)t; - } - - bool converted = false; - char extInfo = 0; - - switch (tType) { - case TSDB_DATA_TYPE_BOOL: - if (sType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(sType)) { - goto _return; - } - if (tmpVar.i64 > 1 ||tmpVar.i64 < 0) { - goto _return; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { - if (converted) { - tVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - continue; - } - - goto _return; - } - pvar = &val; - t = sizeof(val); - break; - } - case TSDB_DATA_TYPE_BINARY: { - if (taosVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - - if (tolower) { - strntolower_s(pvar, pvar, (int32_t)t); - } - break; - } - case TSDB_DATA_TYPE_NCHAR: { - if (taosVariantDump(&tmpVar, tmp, tType, true)) { - goto _return; - } - t = varDataLen(tmp); - pvar = varDataVal(tmp); - break; - } - default: - goto _return; - } - - taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); - taosVariantDestroy(&tmpVar); - memset(&tmpVar, 0, sizeof(tmpVar)); - } - - *q = (void *)pObj; - pObj = NULL; - -_return: - taosVariantDestroy(&tmpVar); - taosHashCleanup(pObj); - tfree(tmp); - - return code; -} - -static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pLeft) { - if((*pLeft)->nodeType == TSQL_NODE_EXPR && (*pLeft)->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation - assert(info->pTable != NULL); - SSchema* schema = (*pLeft)->_node.pLeft->pSchema; - if((*pLeft)->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5); - memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - (*pLeft) = (*pLeft)->_node.pLeft; // -> operation use left as input - }else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) && - (tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){ - SSchema* schema = (*pLeft)->pSchema; - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - uint32_t nullData = TSDB_DATA_JSON_NULL; - jsonKeyMd5(&nullData, INT_BYTES, keyMd5); - memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - }else if(tree->_node.optr == TSDB_RELATION_CONTAINS){ - SSchema* schema = (*pLeft)->pSchema; - if(tree->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5); - memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - } - - return TSDB_CODE_SUCCESS; -} - int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { - tExprNode* pLeft = tree->_node.pLeft; + SOperatorNode *node = (SOperatorNode *)tree; int32_t ret = TSDB_CODE_SUCCESS; - if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret; SFilterFieldId left = {0}, right = {0}; - filterAddFieldFromNode(info, pLeft, &left); + filterAddFieldFromNode(info, node->pLeft, &left); uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); int32_t len = 0; uint32_t uidx = 0; + int32_t code = 0; - if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { - void *data = NULL; - SVariant* var = tree->_node.pRight->pVal; - filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false); - if (data == NULL) { - fltError("failed to convert in param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } + if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { + SNodeListNode *listNode = (SNodeListNode *)node->pRight; + void *fdata = NULL; + SListCell *cell = listNode->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; + + for (int32_t i = 0; i < listNode->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + out.data = malloc(sizeof(int64_t)); - if (taosHashGetSize((SHashObj *)data) <= 0) { - filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); + code = vectorConvertImpl(&in, &out); + if (code) { + fltError("convert from %d to %d failed", in.type, out.type); + tfree(out.data); + FLT_ERR_RET(code); + } - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, uidx); - - taosArrayPush(group, &fgroup); - taosHashCleanup(data); - - return TSDB_CODE_SUCCESS; - } - - void *p = taosHashIterate((SHashObj *)data, NULL); - while(p) { - void *key = taosHashGetDataKey((SHashObj *)data, p); - void *fdata = NULL; - - fdata = malloc(sizeof(int64_t)); - SIMPLE_COPY_VALUES(fdata, key); len = tDataTypes[type].bytes; filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); @@ -1290,16 +1016,11 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { filterAddUnitToGroup(&fgroup, uidx); taosArrayPush(group, &fgroup); - - p = taosHashIterate((SHashObj *)data, p); } - - taosHashCleanup(data); } else { - filterAddFieldFromNode(info, tree->_node.pRight, &right); + filterAddFieldFromNode(info, node->pRight, &right); - ret = filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); - CHK_LRET(ret != TSDB_CODE_SUCCESS, TSDB_CODE_QRY_APP_ERROR, "invalid where condition"); + FLT_ERR_RET(filterAddUnit(info, node->opType, &left, &right, &uidx)); SFilterGroup fgroup = {0}; filterAddUnitToGroup(&fgroup, uidx); @@ -1984,6 +1705,69 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { return TSDB_CODE_SUCCESS; } +int32_t fltGenerateSetFromList(void **data, void *pNode, uint32_t type) { + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + if (NULL == pObj) { + fltError("taosHashInit failed, size:%d", 256); + FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + + int32_t code = 0; + SNodeListNode *nodeList = (SNodeListNode *)pNode; + SListCell *cell = nodeList->pNodeList->pHead; + SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; + int8_t dummy = 0; + int32_t bufLen = 60; + out.data = malloc(bufLen); + int32_t len = 0; + void *buf = NULL; + + for (int32_t i = 0; i < nodeList->pNodeList->length; ++i) { + SValueNode *valueNode = (SValueNode *)cell->pNode; + + if (valueNode->node.resType.type != type) { + in.type = valueNode->node.resType.type; + in.bytes = valueNode->node.resType.bytes; + in.data = nodesGetValueFromNode(valueNode); + + code = vectorConvertImpl(&in, &out); + if (code) { + fltError("convert from %d to %d failed", in.type, out.type); + FLT_ERR_JRET(code); + } + + if (IS_VAR_DATA_TYPE(type)) { + len = varDataLen(out.data); + } else { + len = tDataTypes[type].bytes; + } + + buf = out.data; + } else { + buf = nodesGetValueFromNode(valueNode); + len = valueNode->node.resType.bytes; + } + + if (taosHashPut(pObj, buf, (size_t)len, &dummy, sizeof(dummy))) { + fltError("taosHashPut failed"); + FLT_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + tfree(out.data); + *data = pObj; + + return TSDB_CODE_SUCCESS; + +_return: + + tfree(out.data); + taosHashCleanup(pObj); + + FLT_RET(code); +} int32_t filterInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { @@ -2008,6 +1792,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) { } if (unit->compare.optr == TSDB_RELATION_IN) { + FLT_ERR_RET(fltGenerateSetFromList((void **)&fi->data, fi->desc, type)); filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type, false); if (fi->data == NULL) { fltError("failed to convert in param"); @@ -2739,7 +2524,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ if (info->colRange[m] == NULL) { info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); - info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; + info->colRange[m]->colId = FILTER_GET_COL_FIELD_ID(fi); } assert(colInfo->type == RANGE_TYPE_MR_CTX); @@ -3483,9 +3268,8 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema* sch = fi->desc; - (*fp)(param, sch->colId, &fi->data); + (*fp)(param, FILTER_GET_COL_FIELD_ID(fi), &fi->data); } filterUpdateComUnits(info); @@ -3493,25 +3277,6 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from return TSDB_CODE_SUCCESS; } -int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) { - CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); - CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); - - if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { - return TSDB_CODE_SUCCESS; - } - - for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { - SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema* sch = fi->desc; - - (*fp)(param, sch->colId, sch->name, &fi->data); - } - - filterUpdateComUnits(info); - - return TSDB_CODE_SUCCESS; -} int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { int32_t code = TSDB_CODE_SUCCESS; @@ -3802,6 +3567,95 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { SFltTreeStat *stat = (SFltTreeStat *)pContext; + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { + SLogicConditionNode *node = (SLogicConditionNode *)*pNode; + SListCell *cell = node->pParameterList->pHead; + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + if (NULL == cell || NULL == cell->pNode) { + sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + stat->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + if ((QUERY_NODE_OPERATOR != nodeType(cell->pNode)) && (QUERY_NODE_LOGIC_CONDITION != nodeType(cell->pNode))) { + stat->scalarMode = true; + } + + cell = cell->pNext; + } + + return DEAL_RES_CONTINUE; + } + + if (stat->scalarMode) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_VALUE == nodeType(*pNode)) { + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { + SOperatorNode *node = (SOperatorNode *)*pNode; + if (!FLT_IS_COMPARISON_OPERATOR(node->opType)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (NULL == node->pRight) { + if (sclGetOperatorParamNum(node->opType) > 1) { + sclError("invalid operator, pRight:%d, type:%d", node->pRight, nodeType(node)); + stat->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } else { + if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (nodeType(node->pLeft) == nodeType(node->pRight)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + SNode *t = node->pLeft; + node->pLeft = node->pRight; + node->pRight = t; + } + + if (OP_TYPE_IN == node->opType || QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { + fltError("failed to convert in param"); + stat->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + } + + return DEAL_RES_CONTINUE; + } + + sclError("invalid node type for filter, type:%d", nodeType(*pNode)); + + stat->code = TSDB_CODE_QRY_INVALID_INPUT; + + return DEAL_RES_ERROR; } int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { @@ -3840,10 +3694,11 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); + } else { + info->sclCtx.node = pNode; + FLT_ERR_JRET(fltOptimizeNodes(info, &info->sclCtx.node, &stat)); } - FLT_ERR_JRET(fltOptimizeNodes(info, &pNode, &stat)); - return code; _return: From c9ba7e02ceebdca530e11b22a65a913fe7112174 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Feb 2022 10:46:14 +0800 Subject: [PATCH 8/9] feature/qnode --- include/libs/scalar/filter.h | 5 ++ source/libs/scalar/inc/filterInt.h | 3 +- source/libs/scalar/src/filter.c | 119 +++++++++++++---------------- source/libs/scalar/src/sclvector.c | 15 +++- 4 files changed, 75 insertions(+), 67 deletions(-) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index df83a0de35..4986d50a82 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -24,6 +24,11 @@ extern "C" { typedef struct SFilterInfo SFilterInfo; +typedef struct SFilterColumnParam{ + int32_t numOfCols; + SArray* pDataBlock; +} SFilterColumnParam; + int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index ad40dc5958..6c8e187042 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -309,6 +309,7 @@ typedef struct SFilterInfo { #define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) #define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) #define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) #define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) #define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) @@ -370,4 +371,4 @@ extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); } #endif -#endif // TDENGINE_FILTER_INT_H \ No newline at end of file +#endif // TDENGINE_FILTER_INT_H diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 57c8e8c984..17f83b0c16 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -18,6 +18,7 @@ //#include "queryLog.h" #include "tcompare.h" #include "filterInt.h" +#include "filter.h" OptrStr gOptrStr[] = { {TSDB_RELATION_INVALID, "invalid"}, @@ -1418,8 +1419,8 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SSchema *sch = field->desc; - qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + SColumnRefNode *refNode = (SColumnRefNode *)field->desc; + qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); } qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); @@ -1446,9 +1447,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SSchema *sch = left->desc; + SColumnRefNode *refNode = (SColumnRefNode *)left->desc; if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){ - len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); } if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { @@ -1467,7 +1468,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_CONTAINS){ - sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); } if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { @@ -1769,7 +1770,7 @@ _return: FLT_RET(code); } -int32_t filterInitValFieldData(SFilterInfo *info) { +int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { @@ -1793,7 +1794,6 @@ int32_t filterInitValFieldData(SFilterInfo *info) { if (unit->compare.optr == TSDB_RELATION_IN) { FLT_ERR_RET(fltGenerateSetFromList((void **)&fi->data, fi->desc, type)); - filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type, false); if (fi->data == NULL) { fltError("failed to convert in param"); FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); @@ -2973,47 +2973,6 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } -static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){ - if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){ - uint8_t jsonType = *(char*)colData; - char* realData = POINTER_SHIFT(colData, CHAR_BYTES); - if (jsonType != TSDB_DATA_TYPE_NCHAR){ - *result = false; - }else{ - char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1); - int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData)); - if (len < 0){ - qError("castConvert1 taosUcs4ToMbs error"); - tfree(newColData); - return; - } - varDataSetLen(newColData, len); - tVariant* val = cunit->valData; - char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE); - memcpy(varDataVal(newValData), val->pz, val->nLen); - varDataSetLen(newValData, val->nLen); - *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData); - tfree(newColData); - } - }else if(cunit->optr == TSDB_RELATION_LIKE){ - uint8_t jsonType = *(char*)colData; - char* realData = POINTER_SHIFT(colData, CHAR_BYTES); - if (jsonType != TSDB_DATA_TYPE_NCHAR){ - *result = false; - }else{ - tVariant* val = cunit->valData; - char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1); - memcpy(varDataVal(newValData), val->pz, val->nLen); - varDataSetLen(newValData, val->nLen); - *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData); - tfree(newValData); - } - }else{ - *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); - } -} - bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; @@ -3083,8 +3042,6 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData); } tfree(newColData); - }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ - doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData); }else{ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); } @@ -3145,8 +3102,6 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData); } tfree(newColData); - }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){ - doJsonCompare(cunit, &(*p)[i], colData); }else{ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); } @@ -3174,11 +3129,6 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg return all; } - -FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { - return (*info->func)(info, numOfRows, p, statis, numOfCols); -} - int32_t filterSetExecFunc(SFilterInfo *info) { if (FILTER_ALL_RES(info)) { info->func = filterExecuteImplAll; @@ -3258,7 +3208,8 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + +int32_t fltSetColFieldDataImpl(SFilterInfo *info, void *param, filer_get_col_from_id fp, bool fromColId) { CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); @@ -3269,7 +3220,11 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; - (*fp)(param, FILTER_GET_COL_FIELD_ID(fi), &fi->data); + if (fromColId) { + (*fp)(param, FILTER_GET_COL_FIELD_ID(fi), &fi->data); + } else { + (*fp)(param, FILTER_GET_COL_FIELD_SLOT_ID(fi), &fi->data); + } } filterUpdateComUnits(info); @@ -3292,7 +3247,7 @@ int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) { filterConvertGroupFromArray(info, group); taosArrayDestroy(group); - FLT_ERR_JRET(filterInitValFieldData(info)); + FLT_ERR_JRET(fltInitValFieldData(info)); if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { filterDumpInfoToString(info, "Before preprocess", 0); @@ -3563,8 +3518,6 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) { return TSDB_CODE_SUCCESS; } -################################################################################## - EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { SFltTreeStat *stat = (SFltTreeStat *)pContext; @@ -3641,11 +3594,21 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { node->pRight = t; } - if (OP_TYPE_IN == node->opType || QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { - fltError("failed to convert in param"); + if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { + fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); stat->code = TSDB_CODE_QRY_APP_ERROR; return DEAL_RES_ERROR; - } + } + + if (OP_TYPE_IN != node->opType) { + SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SValueNode *valueNode = (SValueNode *)node->pRight; + int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->dataType.type) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + } } return DEAL_RES_CONTINUE; @@ -3669,6 +3632,32 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) } +int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { + int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; + SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); + if (id == pColInfo->info.colId) { + *data = pColInfo->pData; + break; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + return fltSetColFieldDataImpl(info, param, fp, false); +} + +int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { + return fltSetColFieldDataImpl(info, param, fp, true); +} + + + int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) { int32_t code = 0; SFilterInfo *info = NULL; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 57e9bc0d09..5b2ec7902c 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -515,6 +515,18 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = { /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; +int32_t vectorGetConvertType(int32_t type1, int32_t type2) { + if (type1 == type2) { + return 0; + } + + if (type1 < type2) { + return gConvertTypes[type1][type2]; + } + + return gConvertTypes[type2][type1]; +} + int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* pLeftOut, SScalarParam* pRightOut) { if (pLeft->type == pRight->type) { return TSDB_CODE_SUCCESS; @@ -536,7 +548,8 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p paramOut2 = pLeftOut; } - int8_t type = gConvertTypes[param1->type][param2->type]; + + int8_t type = vectorGetConvertType(param1->type, param2->type); if (0 == type) { return TSDB_CODE_SUCCESS; } From 7c1e4d319d1f2e10945176a97574cdf1146f0b64 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Feb 2022 14:28:56 +0800 Subject: [PATCH 9/9] feature/qnode --- include/libs/scalar/filter.h | 13 +-- include/libs/scalar/scalar.h | 5 + include/util/compare.h | 3 + include/util/thash.h | 4 + source/common/src/tcompare.c | 149 --------------------------- source/libs/scalar/inc/filterInt.h | 1 + source/libs/scalar/src/filter.c | 129 ++++++++++-------------- source/libs/scalar/src/scalar.c | 6 +- source/util/src/compare.c | 156 +++++++++++++++++++++++++++++ source/util/src/thashutil.c | 24 ++--- 10 files changed, 242 insertions(+), 248 deletions(-) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index 4986d50a82..8db74f4587 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -12,16 +12,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_SCALAR_H -#define TDENGINE_SCALAR_H +#ifndef TDENGINE_FILTER_H +#define TDENGINE_FILTER_H #ifdef __cplusplus extern "C" { #endif -#include "function.h" -#include "nodes.h" - typedef struct SFilterInfo SFilterInfo; typedef struct SFilterColumnParam{ @@ -30,12 +27,8 @@ typedef struct SFilterColumnParam{ } SFilterColumnParam; -int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); - - #ifdef __cplusplus } #endif -#endif // TDENGINE_SCALAR_H +#endif // TDENGINE_FILTER_H \ No newline at end of file diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index df83a0de35..4dbdb2e8bb 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -21,12 +21,17 @@ extern "C" { #include "function.h" #include "nodes.h" +#include "querynodes.h" typedef struct SFilterInfo SFilterInfo; int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); +int32_t scalarGetOperatorParamNum(EOperatorType type); + +int32_t vectorGetConvertType(int32_t type1, int32_t type2); +int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut); #ifdef __cplusplus diff --git a/include/util/compare.h b/include/util/compare.h index 0ae85418c7..6305334e02 100644 --- a/include/util/compare.h +++ b/include/util/compare.h @@ -100,6 +100,9 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); +__compar_fn_t getComparFunc(int32_t type, int32_t optr); + + #ifdef __cplusplus } #endif diff --git a/include/util/thash.h b/include/util/thash.h index 3a614a73a6..1f72045a83 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -55,6 +55,8 @@ uint32_t taosIntHash_64(const char *key, uint32_t len); _hash_fn_t taosGetDefaultHashFunction(int32_t type); +_equal_fn_t taosGetDefaultEqualFunction(int32_t type); + typedef struct SHashNode { struct SHashNode *next; uint32_t hashVal; // the hash value of key @@ -258,6 +260,8 @@ void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashRelease(SHashObj *pHashObj, void *p); +void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); + #ifdef __cplusplus } diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c index 0ed490bed0..ef441c97c7 100644 --- a/source/common/src/tcompare.c +++ b/source/common/src/tcompare.c @@ -15,155 +15,6 @@ #include "tcompare.h" -int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); - char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); - memcpy(buf, varDataVal(pLeft), sz); - buf[sz] = 0; - - int32_t ret = patternMatch(pattern, buf, sz, &pInfo); - free(buf); - free(pattern); - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { - return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; -} - -int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { - SPatternCompareInfo pInfo = {'%', '_'}; - - assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); - - wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); - memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); - free(pattern); - - return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; -} - -int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { - return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; -} - -__compar_fn_t getComparFunc(int32_t type, int32_t optr) { - __compar_fn_t comparFn = NULL; - - if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - return setChkInBytes1; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - return setChkInBytes2; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - return setChkInBytes4; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - return setChkInBytes8; - default: - assert(0); - } - } - - if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - return setChkNotInBytes1; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - return setChkNotInBytes2; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_FLOAT: - return setChkNotInBytes4; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - return setChkNotInBytes8; - default: - assert(0); - } - } - - switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; - case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; - case TSDB_DATA_TYPE_BINARY: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternMatch; - } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ - comparFn = compareStrPatternNotMatch; - } else if (optr == TSDB_RELATION_IN) { - comparFn = compareChkInString; - } else if (optr == TSDB_RELATION_NOT_IN) { - comparFn = compareChkNotInString; - } else { /* normal relational comparFn */ - comparFn = compareLenPrefixedStr; - } - - break; - } - - case TSDB_DATA_TYPE_NCHAR: { - if (optr == TSDB_RELATION_MATCH) { - comparFn = compareStrRegexCompMatch; - } else if (optr == TSDB_RELATION_NMATCH) { - comparFn = compareStrRegexCompNMatch; - } else if (optr == TSDB_RELATION_LIKE) { - comparFn = compareWStrPatternMatch; - } else if (optr == TSDB_RELATION_NOT_LIKE) { - comparFn = compareWStrPatternNotMatch; - } else if (optr == TSDB_RELATION_IN) { - comparFn = compareChkInString; - } else if (optr == TSDB_RELATION_NOT_IN) { - comparFn = compareChkNotInString; - } else { - comparFn = compareLenPrefixedWStr; - } - break; - } - - case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; - case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; - - default: - comparFn = compareInt32Val; - break; - } - - return comparFn; -} __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { __compar_fn_t comparFn = NULL; diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 6c8e187042..1dd533c1c5 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -25,6 +25,7 @@ extern "C" { #include "common.h" #include "scalar.h" #include "querynodes.h" +#include "query.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 17f83b0c16..8f8fc25d18 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -36,28 +36,27 @@ OptrStr gOptrStr[] = { {TSDB_RELATION_OR, "or"}, {TSDB_RELATION_NOT, "not"}, {TSDB_RELATION_MATCH, "match"}, - {TSDB_RELATION_NMATCH, "nmatch"}, - {TSDB_RELATION_CONTAINS, "contains"}, + {TSDB_RELATION_NMATCH, "nmatch"} }; bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result >= 0; } bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(maxv, minr); - if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result > 0; } bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result <= 0; } bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { int32_t result = cfunc(minv, maxr); - if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; + //if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; return result < 0; } bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -1299,7 +1298,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext); - FLT_ERR_JRET(ctx.code); + FLT_ERR_JRET(ctx->code); } return DEAL_RES_IGNORE_CHILD; @@ -1316,7 +1315,7 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } - sclError("invalid node type for filter, type:%d", nodeType(pNode)); + fltError("invalid node type for filter, type:%d", nodeType(pNode)); code = TSDB_CODE_QRY_INVALID_INPUT; @@ -1427,11 +1426,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { - SVariant *var = field->desc; - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { + SValueNode *var = (SValueNode *)field->desc; + SDataType *dType = &var->node.resType; + if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); } else { - qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, dType->type, var->datum.i); //TODO } } else if (field->data) { qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO @@ -1448,7 +1448,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnRefNode *refNode = (SColumnRefNode *)left->desc; - if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){ + if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_NMATCH){ len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); } @@ -1467,7 +1467,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_CONTAINS){ + if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_NMATCH){ sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); } @@ -1785,7 +1785,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { uint32_t type = FILTER_UNIT_DATA_TYPE(unit); SFilterField* fi = right; - SVariant* var = fi->desc; + SValueNode* var = (SValueNode *)fi->desc; if (var == NULL) { assert(fi->data != NULL); @@ -1804,20 +1804,23 @@ int32_t fltInitValFieldData(SFilterInfo *info) { continue; } + SDataType *dType = &var->node.resType; + if (type == TSDB_DATA_TYPE_BINARY) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); } else if (type == TSDB_DATA_TYPE_NCHAR) { - size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + size_t len = (dType->type == TSDB_DATA_TYPE_BINARY || dType->type == TSDB_DATA_TYPE_NCHAR) ? dType->bytes : MAX_NUM_STR_SIZE; fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); } else if (type != TSDB_DATA_TYPE_JSON){ - if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE - fi->data = calloc(var->nLen, tDataTypes[type].bytes); - for (int32_t a = 0; a < var->nLen; ++a) { + if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE +/* + fi->data = calloc(dType->bytes, tDataTypes[type].bytes); + for (int32_t a = 0; a < dType->bytes; ++a) { int64_t *v = taosArrayGet(var->arr, a); assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); } - +*/ continue; } else { fi->data = calloc(1, sizeof(int64_t)); @@ -1829,13 +1832,10 @@ int32_t fltInitValFieldData(SFilterInfo *info) { if(type != TSDB_DATA_TYPE_JSON){ bool converted = false; char extInfo = 0; - if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { - if (converted) { - filterHandleValueExtInfo(unit, extInfo); - - continue; - } - qError("dump value to type[%d] failed", type); + SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; + SScalarParam out = {.data = fi->data, .num = 1, .type = type}; + if (vectorConvertImpl(&in, &out)) { + qError("convert value to type[%d] failed", type); return TSDB_CODE_TSC_INVALID_OPERATION; } } @@ -1851,18 +1851,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); - }else if(type == TSDB_DATA_TYPE_JSON && - (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){ - char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0}; - int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData); - if (len < 0){ - qError("filterInitValFieldData taosUcs4ToMbs error 2"); - return TSDB_CODE_FAILED; - } - memcpy(((tVariant*)(fi->desc))->pz, newValData, len); - ((tVariant*)(fi->desc))->nLen = len; } - } return TSDB_CODE_SUCCESS; @@ -1872,8 +1861,6 @@ int32_t fltInitValFieldData(SFilterInfo *info) { bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { int32_t ret = func(left, right); - if(ret == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false; - switch (optr) { case TSDB_RELATION_EQUAL: { return ret == 0; @@ -1899,9 +1886,6 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) case TSDB_RELATION_MATCH: { return ret == 0; } - case TSDB_RELATION_CONTAINS: { - return ret == 0; - } case TSDB_RELATION_NMATCH: { return ret == 0; } @@ -1968,25 +1952,25 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *c } int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { - CHK_JMP(ctx1->status != ctx2->status); - CHK_JMP(ctx1->isnull != ctx2->isnull); - CHK_JMP(ctx1->notnull != ctx2->notnull); - CHK_JMP(ctx1->isrange != ctx2->isrange); + FLT_CHK_JMP(ctx1->status != ctx2->status); + FLT_CHK_JMP(ctx1->isnull != ctx2->isnull); + FLT_CHK_JMP(ctx1->notnull != ctx2->notnull); + FLT_CHK_JMP(ctx1->isrange != ctx2->isrange); SFilterRangeNode *r1 = ctx1->rs; SFilterRangeNode *r2 = ctx2->rs; while (r1 && r2) { - CHK_JMP(r1->ra.sflag != r2->ra.sflag); - CHK_JMP(r1->ra.eflag != r2->ra.eflag); - CHK_JMP(r1->ra.s != r2->ra.s); - CHK_JMP(r1->ra.e != r2->ra.e); + FLT_CHK_JMP(r1->ra.sflag != r2->ra.sflag); + FLT_CHK_JMP(r1->ra.eflag != r2->ra.eflag); + FLT_CHK_JMP(r1->ra.s != r2->ra.s); + FLT_CHK_JMP(r1->ra.e != r2->ra.e); r1 = r1->next; r2 = r2->next; } - CHK_JMP(r1 != r2); + FLT_CHK_JMP(r1 != r2); *equal = true; @@ -2009,11 +1993,11 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colI uint8_t optr = FILTER_UNIT_OPTR(u); filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); - CHK_JMP(*empty); + FLT_CHK_JMP(*empty); if (!FILTER_NO_MERGE_OPTR(optr)) { filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); - CHK_JMP(MR_EMPTY_RES(ctx)); + FLT_CHK_JMP(MR_EMPTY_RES(ctx)); } if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){ gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL; @@ -2214,7 +2198,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); - CHK_JMP(*all); + FLT_CHK_JMP(*all); if (numEqual) { if ((*gRes1)->colNum == 1) { @@ -2234,7 +2218,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++equal2; } - CHK_JMP(equal1 != merNum && equal2 != merNum); + FLT_CHK_JMP(equal1 != merNum && equal2 != merNum); colCtx.colIdx = idx1; colCtx.ctx = ctx; ctx = NULL; @@ -2246,7 +2230,7 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter ++equal1; } - CHK_JMP(equal1 != merNum); + FLT_CHK_JMP(equal1 != merNum); colCtx.colIdx = idx1; colCtx.ctx = ctx; ctx = NULL; @@ -2327,7 +2311,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR assert(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - CHK_JMP(all); + FLT_CHK_JMP(all); if (gRes[n] == NULL) { if (n < ((*gResNum) - 1)) { @@ -2348,7 +2332,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR assert(m < n); filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); - CHK_JMP(all); + FLT_CHK_JMP(all); if (gRes[n] == NULL) { if (n < ((*gResNum) - 1)) { @@ -2503,7 +2487,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ idxs[colNum++] = i; } - CHK_JMP(colNum <= 0); + FLT_CHK_JMP(colNum <= 0); info->colRangeNum = colNum; info->colRange = calloc(colNum, POINTER_BYTES); @@ -2543,7 +2527,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_ --info->colRangeNum; --m; - CHK_JMP(info->colRangeNum <= 0); + FLT_CHK_JMP(info->colRangeNum <= 0); } ++n; @@ -2672,7 +2656,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH - || cunit->optr == TSDB_RELATION_NOT_EQUAL || cunit->optr == TSDB_RELATION_CONTAINS) { + || cunit->optr == TSDB_RELATION_NOT_EQUAL) { continue; } @@ -2973,7 +2957,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows return all; } -bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) { +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; @@ -3175,13 +3159,13 @@ int32_t filterPreprocess(SFilterInfo *info) { filterMergeGroups(info, gRes, &gResNum); if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { - qInfo("Final - FilterInfo: [ALL]"); + fltInfo("Final - FilterInfo: [ALL]"); goto _return; } if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { - qInfo("Final - FilterInfo: [EMPTY]"); + fltInfo("Final - FilterInfo: [EMPTY]"); goto _return; } @@ -3210,9 +3194,6 @@ _return: int32_t fltSetColFieldDataImpl(SFilterInfo *info, void *param, filer_get_col_from_id fp, bool fromColId) { - CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); - CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); - if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { return TSDB_CODE_SUCCESS; } @@ -3392,7 +3373,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { uint8_t raOptr = FILTER_UNIT_OPTR(unit); filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); - CHK_JMP(empty); + FLT_CHK_JMP(empty); if (FILTER_NO_MERGE_OPTR(raOptr)) { continue; @@ -3431,7 +3412,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); } - CHK_JMP(num < 1); + FLT_CHK_JMP(num < 1); SFilterRange tra; filterGetRangeRes(prev, &tra); @@ -3526,7 +3507,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { SListCell *cell = node->pParameterList->pHead; for (int32_t i = 0; i < node->pParameterList->length; ++i) { if (NULL == cell || NULL == cell->pNode) { - sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); + fltError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); stat->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } @@ -3562,8 +3543,8 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (NULL == node->pRight) { - if (sclGetOperatorParamNum(node->opType) > 1) { - sclError("invalid operator, pRight:%d, type:%d", node->pRight, nodeType(node)); + if (scalarGetOperatorParamNum(node->opType) > 1) { + fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node)); stat->code = TSDB_CODE_QRY_APP_ERROR; return DEAL_RES_ERROR; } @@ -3614,7 +3595,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - sclError("invalid node type for filter, type:%d", nodeType(*pNode)); + fltError("invalid node type for filter, type:%d", nodeType(*pNode)); stat->code = TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d13589545a..aa29b02709 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -6,7 +6,7 @@ #include "sclvector.h" #include "sclInt.h" -int32_t sclGetOperatorParamNum(EOperatorType type) { +int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type) { return 1; } @@ -153,7 +153,7 @@ _return: int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScalarCtx *ctx, int32_t *rowNum) { int32_t code = 0; - int32_t paramNum = sclGetOperatorParamNum(node->opType); + int32_t paramNum = scalarGetOperatorParamNum(node->opType); if (NULL == node->pLeft || (paramNum == 2 && NULL == node->pRight)) { sclError("invalid operation node, left:%p, right:%p", node->pLeft, node->pRight); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -299,7 +299,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); - int32_t paramNum = sclGetOperatorParamNum(node->opType); + int32_t paramNum = scalarGetOperatorParamNum(node->opType); SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; diff --git a/source/util/src/compare.c b/source/util/src/compare.c index ab353fcbb3..6124de9459 100644 --- a/source/util/src/compare.c +++ b/source/util/src/compare.c @@ -23,6 +23,8 @@ #include "thash.h" #include "types.h" #include "ulog.h" +#include "tdef.h" +#include "taos.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -418,3 +420,157 @@ int32_t taosArrayCompareString(const void* a, const void* b) { return compareLenPrefixedStr(x, y); } + + +int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); + char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); + free(pattern); + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + +int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { + SPatternCompareInfo pInfo = {'%', '_'}; + + assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); + + wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); + memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); + + int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); + free(pattern); + + return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; +} + +int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { + return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; +} + + +__compar_fn_t getComparFunc(int32_t type, int32_t optr) { + __compar_fn_t comparFn = NULL; + + if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkInBytes8; + default: + assert(0); + } + } + + if (optr == TSDB_RELATION_NOT_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return setChkNotInBytes1; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return setChkNotInBytes2; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return setChkNotInBytes4; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return setChkNotInBytes8; + default: + assert(0); + } + } + + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; + case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_BINARY: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternMatch; + } else if (optr == TSDB_RELATION_NOT_LIKE) { /* wildcard query using like operator */ + comparFn = compareStrPatternNotMatch; + } else if (optr == TSDB_RELATION_IN) { + comparFn = compareChkInString; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = compareChkNotInString; + } else { /* normal relational comparFn */ + comparFn = compareLenPrefixedStr; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = compareStrRegexCompMatch; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = compareStrRegexCompNMatch; + } else if (optr == TSDB_RELATION_LIKE) { + comparFn = compareWStrPatternMatch; + } else if (optr == TSDB_RELATION_NOT_LIKE) { + comparFn = compareWStrPatternNotMatch; + } else if (optr == TSDB_RELATION_IN) { + comparFn = compareChkInString; + } else if (optr == TSDB_RELATION_NOT_IN) { + comparFn = compareChkNotInString; + } else { + comparFn = compareLenPrefixedWStr; + } + break; + } + + case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; + case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + + default: + comparFn = compareInt32Val; + break; + } + + return comparFn; +} + + diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index e91b11026b..2d7f8a5b03 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -164,20 +164,20 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { return fn; } -//int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { -// return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); -//} -// -//int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { -// return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); -//} +int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); +} + +int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); +} _equal_fn_t taosGetDefaultEqualFunction(int32_t type) { _equal_fn_t fn = NULL; -// switch (type) { -// case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; -// case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; -// default: fn = memcmp; break; -// } + switch (type) { + case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; + case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; + default: fn = memcmp; break; + } return fn; }