From 59219933368d3a9f41375e794298ade4a87f7b4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 Nov 2021 11:53:05 +0800 Subject: [PATCH 01/26] [td-10564] support unary scalar function. --- include/util/tcompression.h | 2 +- include/util/tdef.h | 22 +- include/util/tskiplist.h | 2 +- source/libs/executor/src/tarithoperator.c | 411 --------------- .../inc/tbinoperator.h} | 12 +- source/libs/function/inc/texpr.h | 3 - source/libs/function/inc/tscalarfunction.h | 14 +- source/libs/function/inc/tunaryoperator.h | 32 ++ source/libs/function/src/taggfunction.c | 2 +- source/libs/function/src/tbinoperator.c | 484 ++++++++++++++++++ source/libs/function/src/texpr.c | 173 ------- source/libs/function/src/tscalarfunction.c | 163 +++++- source/libs/function/src/tunaryoperator.c | 167 ++++++ source/libs/parser/src/parserUtil.c | 10 +- source/util/src/thash.c | 1 + source/util/src/thashutil.c | 2 +- 16 files changed, 874 insertions(+), 626 deletions(-) delete mode 100644 source/libs/executor/src/tarithoperator.c rename source/libs/{executor/inc/tarithoperator.h => function/inc/tbinoperator.h} (62%) create mode 100644 source/libs/function/inc/tunaryoperator.h create mode 100644 source/libs/function/src/tbinoperator.c create mode 100644 source/libs/function/src/tunaryoperator.c diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 1e76b5198a..4cb1193578 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "tdef.h" +#include "taos.h" #include "tutil.h" #define COMP_OVERFLOW_BYTES 2 diff --git a/include/util/tdef.h b/include/util/tdef.h index 80cd3cf8b8..7a684ba2e1 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -134,6 +134,13 @@ do { \ #define TSDB_BINARY_OP_REMAINDER 4004 #define TSDB_BINARY_OP_CONCAT 4005 +#define TSDB_UNARY_OP_CEIL 4500 +#define TSDB_UNARY_OP_FLOOR 4501 +#define TSDB_UNARY_OP_ABS 4502 +#define TSDB_UNARY_OP_ROUND 4503 + +#define TSDB_UNARY_OP_LEN 4600 + #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) @@ -366,21 +373,6 @@ do { \ #define TSDB_MAX_DISKS_PER_TIER 16 #define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string -#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte -#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes -#define TSDB_DATA_TYPE_UINT 13 // 4 bytes -#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK }; diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 02db8cb534..823a6f247f 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -//#include "tdef.h" +#include "taos.h" #include "tarray.h" #include "tfunctional.h" diff --git a/source/libs/executor/src/tarithoperator.c b/source/libs/executor/src/tarithoperator.c deleted file mode 100644 index 6789eb5916..0000000000 --- a/source/libs/executor/src/tarithoperator.c +++ /dev/null @@ -1,411 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" - -#include "ttypes.h" -#include "tarithoperator.h" -#include "tcompare.h" - -//GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[i])); - -void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - - *pOutput = (double)pLeft[i] + pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - - *pOutput = (double)pLeft[0] + pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - SET_DOUBLE_NULL(pOutput); - continue; - } - *pOutput = (double)pLeft[i] + pRight[0]; - } - } -} - -typedef double (*_arithmetic_getVectorDoubleValue_fn_t)(void *src, int32_t index); - -double getVectorDoubleValue_TINYINT(void *src, int32_t index) { - return (double)*((int8_t *)src + index); -} -double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { - return (double)*((uint8_t *)src + index); -} -double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { - return (double)*((int16_t *)src + index); -} -double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { - return (double)*((uint16_t *)src + index); -} -double getVectorDoubleValue_INT(void *src, int32_t index) { - return (double)*((int32_t *)src + index); -} -double getVectorDoubleValue_UINT(void *src, int32_t index) { - return (double)*((uint32_t *)src + index); -} -double getVectorDoubleValue_BIGINT(void *src, int32_t index) { - return (double)*((int64_t *)src + index); -} -double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { - return (double)*((uint64_t *)src + index); -} -double getVectorDoubleValue_FLOAT(void *src, int32_t index) { - return (double)*((float *)src + index); -} -double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { - return (double)*((double *)src + index); -} -_arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { - _arithmetic_getVectorDoubleValue_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorDoubleValue_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorDoubleValue_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorDoubleValue_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorDoubleValue_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorDoubleValue_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorDoubleValue_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorDoubleValue_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorDoubleValue_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorDoubleValue_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorDoubleValue_DOUBLE; - }else { - assert(0); - } - return p; -} - - -typedef void* (*_arithmetic_getVectorValueAddr_fn_t)(void *src, int32_t index); - -void* getVectorValueAddr_TINYINT(void *src, int32_t index) { - return (void*)((int8_t *)src + index); -} -void* getVectorValueAddr_UTINYINT(void *src, int32_t index) { - return (void*)((uint8_t *)src + index); -} -void* getVectorValueAddr_SMALLINT(void *src, int32_t index) { - return (void*)((int16_t *)src + index); -} -void* getVectorValueAddr_USMALLINT(void *src, int32_t index) { - return (void*)((uint16_t *)src + index); -} -void* getVectorValueAddr_INT(void *src, int32_t index) { - return (void*)((int32_t *)src + index); -} -void* getVectorValueAddr_UINT(void *src, int32_t index) { - return (void*)((uint32_t *)src + index); -} -void* getVectorValueAddr_BIGINT(void *src, int32_t index) { - return (void*)((int64_t *)src + index); -} -void* getVectorValueAddr_UBIGINT(void *src, int32_t index) { - return (void*)((uint64_t *)src + index); -} -void* getVectorValueAddr_FLOAT(void *src, int32_t index) { - return (void*)((float *)src + index); -} -void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { - return (void*)((double *)src + index); -} - -_arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { - _arithmetic_getVectorValueAddr_fn_t p = NULL; - if(srcType==TSDB_DATA_TYPE_TINYINT) { - p = getVectorValueAddr_TINYINT; - }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { - p = getVectorValueAddr_UTINYINT; - }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { - p = getVectorValueAddr_SMALLINT; - }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { - p = getVectorValueAddr_USMALLINT; - }else if(srcType==TSDB_DATA_TYPE_INT) { - p = getVectorValueAddr_INT; - }else if(srcType==TSDB_DATA_TYPE_UINT) { - p = getVectorValueAddr_UINT; - }else if(srcType==TSDB_DATA_TYPE_BIGINT) { - p = getVectorValueAddr_BIGINT; - }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { - p = getVectorValueAddr_UBIGINT; - }else if(srcType==TSDB_DATA_TYPE_FLOAT) { - p = getVectorValueAddr_FLOAT; - }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { - p = getVectorValueAddr_DOUBLE; - }else { - assert(0); - } - return p; -} - -void vectorAdd(void *left, int32_t len1, int32_t _left_type, void *right, int32_t len2, int32_t _right_type, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - double *output=(double*)out; - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(_left_type); - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(_right_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(_left_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(_right_type); - - if ((len1) == (len2)) { - for (; i < (len2) && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) + getVectorDoubleValueFnRight(right,i)); - } - } else if ((len1) == 1) { - for (; i >= 0 && i < (len2); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,0), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,0) + getVectorDoubleValueFnRight(right,i)); - } - } else if ((len2) == 1) { - for (; i >= 0 && i < (len1); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,0), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) + getVectorDoubleValueFnRight(right,0)); - } - } -} -void vectorSub(void *left, int32_t len1, int32_t _left_type, void *right, int32_t len2, int32_t _right_type, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - double *output=(double*)out; - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(_left_type); - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(_right_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(_left_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(_right_type); - - if ((len1) == (len2)) { - for (; i < (len2) && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) - getVectorDoubleValueFnRight(right,i)); - } - } else if ((len1) == 1) { - for (; i >= 0 && i < (len2); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,0), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,0) - getVectorDoubleValueFnRight(right,i)); - } - } else if ((len2) == 1) { - for (; i >= 0 && i < (len1); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,0), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) - getVectorDoubleValueFnRight(right,0)); - } - } -} -void vectorMultiply(void *left, int32_t len1, int32_t _left_type, void *right, int32_t len2, int32_t _right_type, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - double *output=(double*)out; - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(_left_type); - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(_right_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(_left_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(_right_type); - - if ((len1) == (len2)) { - for (; i < (len2) && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) * getVectorDoubleValueFnRight(right,i)); - } - } else if ((len1) == 1) { - for (; i >= 0 && i < (len2); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,0), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,0) * getVectorDoubleValueFnRight(right,i)); - } - } else if ((len2) == 1) { - for (; i >= 0 && i < (len1); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,0), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) * getVectorDoubleValueFnRight(right,0)); - } - } -} -void vectorDivide(void *left, int32_t len1, int32_t _left_type, void *right, int32_t len2, int32_t _right_type, void *out, int32_t _ord) { - int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - double *output=(double*)out; - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(_left_type); - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(_right_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(_left_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(_right_type); - - if ((len1) == (len2)) { - for (; i < (len2) && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - double v, u = 0.0; - GET_TYPED_DATA(v, double, _right_type, getVectorValueAddrFnRight(right,i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) /getVectorDoubleValueFnRight(right,i)); - } - } else if ((len1) == 1) { - for (; i >= 0 && i < (len2); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,0), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - double v, u = 0.0; - GET_TYPED_DATA(v, double, _right_type, getVectorValueAddrFnRight(right,i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,0) /getVectorDoubleValueFnRight(right,i)); - } - } else if ((len2) == 1) { - for (; i >= 0 && i < (len1); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,0), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - double v, u = 0.0; - GET_TYPED_DATA(v, double, _right_type, getVectorValueAddrFnRight(right,0)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) /getVectorDoubleValueFnRight(right,0)); - } - } -} -void vectorRemainder(void *left, int32_t len1, int32_t _left_type, void *right, int32_t len2, int32_t _right_type, void *out, int32_t _ord) { - int32_t i = (_ord == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; - int32_t step = (_ord == TSDB_ORDER_ASC) ? 1 : -1; - double *output=(double*)out; - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(_left_type); - _arithmetic_getVectorValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(_right_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(_left_type); - _arithmetic_getVectorDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(_right_type); - - if (len1 == (len2)) { - for (; i >= 0 && i < (len2); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - double v, u = 0.0; - GET_TYPED_DATA(v, double, _right_type, getVectorValueAddrFnRight(right,i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) - ((int64_t)(getVectorDoubleValueFnLeft(left,i) / getVectorDoubleValueFnRight(right,i))) * getVectorDoubleValueFnRight(right,i)); - } - } else if (len1 == 1) { - for (; i >= 0 && i < (len2); i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,0), _left_type) || isNull(getVectorValueAddrFnRight(right,i), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - double v, u = 0.0; - GET_TYPED_DATA(v, double, _right_type, getVectorValueAddrFnRight(right,i)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,0) - ((int64_t)(getVectorDoubleValueFnLeft(left,0) / getVectorDoubleValueFnRight(right,i))) * getVectorDoubleValueFnRight(right,i)); - } - } else if ((len2) == 1) { - for (; i >= 0 && i < len1; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(left,i), _left_type) || isNull(getVectorValueAddrFnRight(right,0), _right_type)) { - SET_DOUBLE_NULL(output); - continue; - } - double v, u = 0.0; - GET_TYPED_DATA(v, double, _right_type, getVectorValueAddrFnRight(right,0)); - if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); - continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(left,i) - ((int64_t)(getVectorDoubleValueFnLeft(left,i) / getVectorDoubleValueFnRight(right,0))) * getVectorDoubleValueFnRight(right,0)); - } - } -} - -_arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr) { - switch (arithmeticOptr) { - case TSDB_BINARY_OP_ADD: - return vectorAdd; - case TSDB_BINARY_OP_SUBTRACT: - return vectorSub; - case TSDB_BINARY_OP_MULTIPLY: - return vectorMultiply; - case TSDB_BINARY_OP_DIVIDE: - return vectorDivide; - case TSDB_BINARY_OP_REMAINDER: - return vectorRemainder; - default: - assert(0); - return NULL; - } -} diff --git a/source/libs/executor/inc/tarithoperator.h b/source/libs/function/inc/tbinoperator.h similarity index 62% rename from source/libs/executor/inc/tarithoperator.h rename to source/libs/function/inc/tbinoperator.h index e47cb5c1cb..65b9ce1bde 100644 --- a/source/libs/executor/inc/tarithoperator.h +++ b/source/libs/function/inc/tbinoperator.h @@ -13,20 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_QARITHMETICOPERATOR_H_ -#define _TD_COMMON_QARITHMETICOPERATOR_H_ +#ifndef _TD_COMMON_BIN_SCALAR_OPERATOR_H_ +#define _TD_COMMON_BIN_SCALAR_OPERATOR_H_ #ifdef __cplusplus extern "C" { #endif -typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, - int32_t rightType, void *output, int32_t order); +#include "tscalarfunction.h" -_arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr); +typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order); +_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_QARITHMETICOPERATOR_H_*/ +#endif /*_TD_COMMON_BIN_SCALAR_OPERATOR_H_*/ diff --git a/source/libs/function/inc/texpr.h b/source/libs/function/inc/texpr.h index 3319095470..bb5c72f7d1 100644 --- a/source/libs/function/inc/texpr.h +++ b/source/libs/function/inc/texpr.h @@ -65,9 +65,6 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond); bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); -void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*cb)(void *, const char*, int32_t)); - void buildFilterSetFromBinary(void **q, const char *buf, int32_t len); #ifdef __cplusplus diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/function/inc/tscalarfunction.h index 9c87a42cfc..eafec75d87 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/function/inc/tscalarfunction.h @@ -21,14 +21,22 @@ extern "C" { #include "function.h" +typedef struct SScalarFuncParam { + void* data; + int32_t num; + int32_t type; + int32_t bytes; +} SScalarFuncParam; + extern struct SScalarFunctionInfo scalarFunc[1]; #define FUNCTION_CEIL 38 #define FUNCTION_FLOOR 39 #define FUNCTION_ROUND 40 -#define FUNCTION_MAVG 41 -#define FUNCTION_CSUM 42 -#define FUNCCTION_CONCAT 43 +#define FUNCTION_CONCAT 41 + +int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, + void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); #ifdef __cplusplus } diff --git a/source/libs/function/inc/tunaryoperator.h b/source/libs/function/inc/tunaryoperator.h new file mode 100644 index 0000000000..27784e8487 --- /dev/null +++ b/source/libs/function/inc/tunaryoperator.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_UNARY_SCALAR_OPERATOR_H_ +#define _TD_COMMON_UNARY_SCALAR_OPERATOR_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tscalarfunction.h" + +typedef void (*_unary_scalar_fn_t)(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput); +_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_BIN_SCALAR_OPERATOR_H_*/ diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index f35ffe2468..5d3f93f9d6 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -3197,7 +3197,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; -// arithmeticTreeTraverse(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); +// evaluateExprNodeTree(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c new file mode 100644 index 0000000000..902c0054a0 --- /dev/null +++ b/source/libs/function/src/tbinoperator.c @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "ttypes.h" +#include "tbinoperator.h" +#include "tcompare.h" + +//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i])); + +void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { + int32_t *pLeft = (int32_t *)left; + int32_t *pRight = (int32_t *)right; + double * pOutput = (double *)output; + + int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; + int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; + + if (numLeft == numRight) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[i] + pRight[i]; + } + } else if (numLeft == 1) { + for (; i >= 0 && i < numRight; i += step, pOutput += 1) { + if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + + *pOutput = (double)pLeft[0] + pRight[i]; + } + } else if (numRight == 1) { + for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { + if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { + SET_DOUBLE_NULL(pOutput); + continue; + } + *pOutput = (double)pLeft[i] + pRight[0]; + } + } +} + +typedef double (*_getDoubleValue_fn_t)(void *src, int32_t index); + +double getVectorDoubleValue_TINYINT(void *src, int32_t index) { + return (double)*((int8_t *)src + index); +} +double getVectorDoubleValue_UTINYINT(void *src, int32_t index) { + return (double)*((uint8_t *)src + index); +} +double getVectorDoubleValue_SMALLINT(void *src, int32_t index) { + return (double)*((int16_t *)src + index); +} +double getVectorDoubleValue_USMALLINT(void *src, int32_t index) { + return (double)*((uint16_t *)src + index); +} +double getVectorDoubleValue_INT(void *src, int32_t index) { + return (double)*((int32_t *)src + index); +} +double getVectorDoubleValue_UINT(void *src, int32_t index) { + return (double)*((uint32_t *)src + index); +} +double getVectorDoubleValue_BIGINT(void *src, int32_t index) { + return (double)*((int64_t *)src + index); +} +double getVectorDoubleValue_UBIGINT(void *src, int32_t index) { + return (double)*((uint64_t *)src + index); +} +double getVectorDoubleValue_FLOAT(void *src, int32_t index) { + return (double)*((float *)src + index); +} +double getVectorDoubleValue_DOUBLE(void *src, int32_t index) { + return (double)*((double *)src + index); +} +_getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) { + _getDoubleValue_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorDoubleValue_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorDoubleValue_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorDoubleValue_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorDoubleValue_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorDoubleValue_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorDoubleValue_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorDoubleValue_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorDoubleValue_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorDoubleValue_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorDoubleValue_DOUBLE; + }else { + assert(0); + } + return p; +} + + +typedef void* (*_getValueAddr_fn_t)(void *src, int32_t index); + +void* getVectorValueAddr_TINYINT(void *src, int32_t index) { + return (void*)((int8_t *)src + index); +} +void* getVectorValueAddr_UTINYINT(void *src, int32_t index) { + return (void*)((uint8_t *)src + index); +} +void* getVectorValueAddr_SMALLINT(void *src, int32_t index) { + return (void*)((int16_t *)src + index); +} +void* getVectorValueAddr_USMALLINT(void *src, int32_t index) { + return (void*)((uint16_t *)src + index); +} +void* getVectorValueAddr_INT(void *src, int32_t index) { + return (void*)((int32_t *)src + index); +} +void* getVectorValueAddr_UINT(void *src, int32_t index) { + return (void*)((uint32_t *)src + index); +} +void* getVectorValueAddr_BIGINT(void *src, int32_t index) { + return (void*)((int64_t *)src + index); +} +void* getVectorValueAddr_UBIGINT(void *src, int32_t index) { + return (void*)((uint64_t *)src + index); +} +void* getVectorValueAddr_FLOAT(void *src, int32_t index) { + return (void*)((float *)src + index); +} +void* getVectorValueAddr_DOUBLE(void *src, int32_t index) { + return (void*)((double *)src + index); +} + +_getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { + _getValueAddr_fn_t p = NULL; + if(srcType==TSDB_DATA_TYPE_TINYINT) { + p = getVectorValueAddr_TINYINT; + }else if(srcType==TSDB_DATA_TYPE_UTINYINT) { + p = getVectorValueAddr_UTINYINT; + }else if(srcType==TSDB_DATA_TYPE_SMALLINT) { + p = getVectorValueAddr_SMALLINT; + }else if(srcType==TSDB_DATA_TYPE_USMALLINT) { + p = getVectorValueAddr_USMALLINT; + }else if(srcType==TSDB_DATA_TYPE_INT) { + p = getVectorValueAddr_INT; + }else if(srcType==TSDB_DATA_TYPE_UINT) { + p = getVectorValueAddr_UINT; + }else if(srcType==TSDB_DATA_TYPE_BIGINT) { + p = getVectorValueAddr_BIGINT; + }else if(srcType==TSDB_DATA_TYPE_UBIGINT) { + p = getVectorValueAddr_UBIGINT; + }else if(srcType==TSDB_DATA_TYPE_FLOAT) { + p = getVectorValueAddr_FLOAT; + }else if(srcType==TSDB_DATA_TYPE_DOUBLE) { + p = getVectorValueAddr_DOUBLE; + }else { + assert(0); + } + return p; +} + +void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + } + } +} + +void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); + } + } +} +void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); + } + } +} + +void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double *output=(double*)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); + } + } else if (pLeft->num == 1) { + double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); + } + } else if (pRight->num == 1) { + double right = getVectorDoubleValueFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(&right, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); + } + } +} + +void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + double * output = (double *)out; + _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); + _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); + _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double left = getVectorDoubleValueFnLeft(pLeft->data, i); + double right = getVectorDoubleValueFnRight(pRight->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } else if (pLeft->num == 1) { + double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + + for (; i >= 0 && i < pRight->num; i += step, output += 1) { + if (isNull(&left, pLeft->type) || + isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double right = getVectorDoubleValueFnRight(pRight->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } else if (pRight->num == 1) { + double right = getVectorDoubleValueFnRight(pRight->data, 0); + + for (; i >= 0 && i < pLeft->num; i += step, output += 1) { + if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || + isNull(&right, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + double v, u = 0.0; + GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); + if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { + SET_DOUBLE_NULL(output); + continue; + } + + double left = getVectorDoubleValueFnLeft(pLeft->data, i); + SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + } + } +} + +void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { + int32_t len = pLeft->bytes + pRight->bytes; + + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + + char *output = (char *)out; + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step, output += len) { + char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); + char* right = POINTER_SHIFT(pRight->data, pRight->bytes * i); + + if (isNull(left, pLeft->type) || isNull(right, pRight->type)) { + setVardataNull(output, TSDB_DATA_TYPE_BINARY); + continue; + } + + // todo define a macro + memcpy(varDataVal(output), varDataVal(left), varDataLen(left)); + memcpy(varDataVal(output) + varDataLen(left), varDataVal(right), varDataLen(right)); + varDataSetLen(output, varDataLen(left) + varDataLen(right)); + } + } else if (pLeft->num == 1) { + for (; i >= 0 && i < pRight->num; i += step, output += len) { + char *right = POINTER_SHIFT(pRight->data, pRight->bytes * i); + if (isNull(pLeft->data, pLeft->type) || isNull(right, pRight->type)) { + setVardataNull(output, TSDB_DATA_TYPE_BINARY); + continue; + } + + memcpy(varDataVal(output), varDataVal(pLeft->data), varDataLen(pLeft->data)); + memcpy(varDataVal(output) + varDataLen(pLeft->data), varDataVal(right), varDataLen(right)); + varDataSetLen(output, varDataLen(pLeft->data) + varDataLen(right)); + } + } else if (pRight->num == 1) { + for (; i >= 0 && i < pLeft->num; i += step, output += len) { + char* left = POINTER_SHIFT(pLeft->data, pLeft->bytes * i); + if (isNull(left, pLeft->type) || isNull(pRight->data, pRight->type)) { + SET_DOUBLE_NULL(output); + continue; + } + + memcpy(varDataVal(output), varDataVal(left), varDataLen(pRight->data)); + memcpy(varDataVal(output) + varDataLen(left), varDataVal(pRight->data), varDataLen(pRight->data)); + varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); + } + } + +} + +_bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { + switch (binFunctionId) { + case TSDB_BINARY_OP_ADD: + return vectorAdd; + case TSDB_BINARY_OP_SUBTRACT: + return vectorSub; + case TSDB_BINARY_OP_MULTIPLY: + return vectorMultiply; + case TSDB_BINARY_OP_DIVIDE: + return vectorDivide; + case TSDB_BINARY_OP_REMAINDER: + return vectorRemainder; + case TSDB_BINARY_OP_CONCAT: + return vectorConcat; + default: + assert(0); + return NULL; + } +} diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index f02e5e77f5..2710b07bc1 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -25,7 +25,6 @@ #include "thash.h" #include "tskiplist.h" #include "texpr.h" -//#include "tarithoperator.h" #include "tvariant.h" //static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { @@ -41,71 +40,6 @@ // } //} -static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { - switch(type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT:{ - int8_t* p = (int8_t*) dest; - int8_t* pSrc = (int8_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT:{ - int16_t* p = (int16_t*) dest; - int16_t* pSrc = (int16_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: { - int32_t* p = (int32_t*) dest; - int32_t* pSrc = (int32_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: { - int64_t* p = (int64_t*) dest; - int64_t* pSrc = (int64_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) dest; - float* pSrc = (float*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) dest; - double* pSrc = (double*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - default: assert(0); - } -} - static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { @@ -182,114 +116,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp return param->nodeFilterFn(pItem, pExpr->_node.info); } -void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*getSourceDataBlock)(void *, const char*, int32_t)) { - if (pExprs == NULL) { - return; - } -#if 0 - tExprNode *pLeft = pExprs->_node.pLeft; - tExprNode *pRight = pExprs->_node.pRight; - /* the left output has result from the left child syntax tree */ - char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows); - if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE) { - arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock); - } - - // the right output has result from the right child syntax tree - char *pRightOutput = malloc(sizeof(int64_t) * numOfRows); - char *pdata = malloc(sizeof(int64_t) * numOfRows); - - if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { - arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); - } - - if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE) { - if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { - /* - * exprLeft + exprRight - * the type of returned value of one expression is always double float precious - */ - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); - - } else if (pRight->nodeType == TEXPR_COL_NODE) { // exprLeft + columnRight - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - - // set input buffer - char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - if (order == TSDB_ORDER_DESC) { - reverseCopy(pdata, pInputData, pRight->pSchema->type, numOfRows); - OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pdata, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); - } else { - OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); - } - - } else if (pRight->nodeType == TEXPR_VALUE_NODE) { // exprLeft + 12 - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, &pRight->pVal->i, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); - } - } else if (pLeft->nodeType == TEXPR_COL_NODE) { - // column data specified on left-hand-side - char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); - if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { // columnLeft + expr2 - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - - if (order == TSDB_ORDER_DESC) { - reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); - OperatorFn(pdata, numOfRows, pLeft->pSchema->type, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); - } else { - OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); - } - - } else if (pRight->nodeType == TEXPR_COL_NODE) { // columnLeft + columnRight - // column data specified on right-hand-side - char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - - // both columns are descending order, do not reverse the source data - OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, order); - } else if (pRight->nodeType == TEXPR_VALUE_NODE) { // columnLeft + 12 - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - - if (order == TSDB_ORDER_DESC) { - reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); - OperatorFn(pdata, numOfRows, pLeft->pSchema->type, &pRight->pVal->i, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); - } else { - OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, &pRight->pVal->i, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); - } - } - } else { - // column data specified on left-hand-side - if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { // 12 + expr2 - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - OperatorFn(&pLeft->pVal->i, 1, pLeft->pVal->nType, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC); - - } else if (pRight->nodeType == TEXPR_COL_NODE) { // 12 + columnRight - // column data specified on right-hand-side - char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - - if (order == TSDB_ORDER_DESC) { - reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows); - OperatorFn(&pLeft->pVal->i, 1, pLeft->pVal->nType, pdata, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); - } else { - OperatorFn(&pLeft->pVal->i, 1, pLeft->pVal->nType, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC); - } - - } else if (pRight->nodeType == TEXPR_VALUE_NODE) { // 12 + 12 - _arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr); - OperatorFn(&pLeft->pVal->i, 1, pLeft->pVal->nType, &pRight->pVal->i, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC); - } - } - - tfree(pdata); - tfree(pLeftOutput); - tfree(pRightOutput); -#endif - -} static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { tbufWriteUint8(bw, expr->nodeType); diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 3e19a09a4d..8da1a9d582 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -1,4 +1,165 @@ #include "tscalarfunction.h" +#include "tbinoperator.h" +#include "tunaryoperator.h" + +static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { + switch(type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT:{ + int8_t* p = (int8_t*) dest; + int8_t* pSrc = (int8_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + return; + } + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT:{ + int16_t* p = (int16_t*) dest; + int16_t* pSrc = (int16_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + return; + } + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + int32_t* p = (int32_t*) dest; + int32_t* pSrc = (int32_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + return; + } + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: { + int64_t* p = (int64_t*) dest; + int64_t* pSrc = (int64_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + return; + } + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) dest; + float* pSrc = (float*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + return; + } + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) dest; + double* pSrc = (double*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + return; + } + default: assert(0); + } +} + +static void setScalarFuncParam(SScalarFuncParam* param, int32_t type, int32_t bytes, void* pInput, int32_t numOfRows) { + param->bytes = bytes; + param->type = type; + param->num = numOfRows; + param->data = pInput; +} + +int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, + char* (*getSourceDataBlock)(void*, const char*, int32_t)) { + if (pExprs == NULL) { + return 0; + } + + tExprNode* pLeft = pExprs->_node.pLeft; + tExprNode* pRight = pExprs->_node.pRight; + + /* the left output has result from the left child syntax tree */ + SScalarFuncParam leftOutput = {0}; + SScalarFuncParam rightOutput = {0}; + + leftOutput.data = malloc(sizeof(int64_t) * numOfRows); + + if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { + evaluateExprNodeTree(pLeft, numOfRows, &leftOutput, param, getSourceDataBlock); + } + + // the right output has result from the right child syntax tree + rightOutput.data = malloc(sizeof(int64_t) * numOfRows); + + if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { + evaluateExprNodeTree(pRight, numOfRows, &rightOutput, param, getSourceDataBlock); + } + + if (pExprs->nodeType == TEXPR_BINARYEXPR_NODE) { + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(pExprs->_node.optr); + + SScalarFuncParam left = {0}, right = {0}; + + if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { + setScalarFuncParam(&left, leftOutput.type, leftOutput.bytes, leftOutput.data, leftOutput.num); + } else if (pLeft->nodeType == TEXPR_COL_NODE) { + SSchema* pschema = pLeft->pSchema; + char* pLeftInputData = getSourceDataBlock(param, pschema->name, pschema->colId); + setScalarFuncParam(&right, pschema->type, pschema->bytes, pLeftInputData, numOfRows); + } else if (pLeft->nodeType == TEXPR_VALUE_NODE) { + SVariant* pVar = pRight->pVal; + setScalarFuncParam(&left, pVar->nType, pVar->nLen, &pVar->i, 1); + } else { + assert(0); + } + + if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) { + setScalarFuncParam(&right, rightOutput.type, rightOutput.bytes, rightOutput.data, rightOutput.num); + } else if (pRight->nodeType == TEXPR_COL_NODE) { // exprLeft + columnRight + SSchema* pschema = pRight->pSchema; + char* pInputData = getSourceDataBlock(param, pschema->name, pschema->colId); + setScalarFuncParam(&right, pschema->type, pschema->bytes, pInputData, numOfRows); + } else if (pRight->nodeType == TEXPR_VALUE_NODE) { // exprLeft + 12 + SVariant* pVar = pRight->pVal; + setScalarFuncParam(&right, pVar->nType, pVar->nLen, &pVar->i, 1); + } else { + assert(0); + } + + OperatorFn(&left, &right, pOutput->data, TSDB_ORDER_ASC); + + // Set the result info + setScalarFuncParam(pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double), pOutput->data, numOfRows); + } else if (pExprs->nodeType == TEXPR_UNARYEXPR_NODE) { + _unary_scalar_fn_t OperatorFn = getUnaryScalarOperatorFn(pExprs->_node.optr); + SScalarFuncParam left = {0}; + + if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { + setScalarFuncParam(&left, leftOutput.type, leftOutput.bytes, leftOutput.data, leftOutput.num); + } else if (pLeft->nodeType == TEXPR_COL_NODE) { + SSchema* pschema = pLeft->pSchema; + char* pLeftInputData = getSourceDataBlock(param, pschema->name, pschema->colId); + setScalarFuncParam(&left, pschema->type, pschema->bytes, pLeftInputData, numOfRows); + } else if (pLeft->nodeType == TEXPR_VALUE_NODE) { + SVariant* pVar = pLeft->pVal; + setScalarFuncParam(&left, pVar->nType, pVar->nLen, &pVar->i, 1); + } else { + assert(0); + } + + OperatorFn(&left, pOutput); + } + + tfree(leftOutput.data); + tfree(rightOutput.data); + + return 0; +} SScalarFunctionInfo scalarFunc[1] = { { @@ -6,5 +167,3 @@ SScalarFunctionInfo scalarFunc[1] = { }, }; - - diff --git a/source/libs/function/src/tunaryoperator.c b/source/libs/function/src/tunaryoperator.c new file mode 100644 index 0000000000..dc9727d841 --- /dev/null +++ b/source/libs/function/src/tunaryoperator.c @@ -0,0 +1,167 @@ +#include "tunaryoperator.h" + +static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { + dst->type = src->type; + dst->bytes = src->bytes; + dst->num = src->num; +} + +static void tceil(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = ceilf(p[i]); + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*)pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = ceil(p[i]); + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void tfloor(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = floorf(p[i]); + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*) pOutput->data; + + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = floor(p[i]); + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void _tabs(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_TINYINT: { + int8_t* p = (int8_t*) pLeft->data; + int8_t* out = (int8_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* p = (int16_t*) pLeft->data; + int16_t* out = (int16_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_INT: { + int32_t* p = (int32_t*) pLeft->data; + int32_t* out = (int32_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* p = (int64_t*) pLeft->data; + int64_t* out = (int64_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void tround(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = roundf(p[i]); + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = round(p[i]); + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void tlen(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + int64_t* out = (int64_t*) pOutput->data; + char* s = pLeft->data; + + for(int32_t i = 0; i < pLeft->num; ++i) { + out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); + } +} + +_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { + switch(operator) { + case TSDB_UNARY_OP_CEIL: + return tceil; + case TSDB_UNARY_OP_FLOOR: + return tfloor; + case TSDB_UNARY_OP_ROUND: + return tround; + case TSDB_UNARY_OP_ABS: + return _tabs; + case TSDB_UNARY_OP_LEN: + return tlen; + default: + assert(0); + } +} diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index c970283ca7..bbe94e7c78 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1175,14 +1175,6 @@ int32_t queryInfoCopy(SQueryStmtInfo* pQueryInfo, const SQueryStmtInfo* pSrc) { goto _error; } -// if (pQueryInfo->arithmeticOnAgg) { -// pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES); -// if (copyAllExprInfo(pQueryInfo->exprList1, pSrc->exprList1, true) != 0) { -// code = TSDB_CODE_TSC_OUT_OF_MEMORY; -// goto _error; -// } -// } - columnListCopyAll(pQueryInfo->colList, pSrc->colList); copyFieldInfo(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList); @@ -1615,9 +1607,9 @@ uint32_t convertRelationalOperator(SToken *pToken) { return TSDB_RELATION_OR; case TK_EQ: return TSDB_RELATION_EQUAL; + case TK_PLUS: return TSDB_BINARY_OP_ADD; - case TK_MINUS: return TSDB_BINARY_OP_SUBTRACT; case TK_STAR: diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 448fc0ab2e..bfc6b47fa6 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -16,6 +16,7 @@ #include "os.h" #include "thash.h" #include "ulog.h" +#include "taos.h" #include "tdef.h" #define EXT_SIZE 1024 diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 7deb4d5a78..e91b11026b 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -16,7 +16,7 @@ #include "os.h" #include "thash.h" #include "compare.h" -#include "tdef.h" +#include "taos.h" #include "types.h" #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) From 01b178eb0e8e1b3fbfe1c2dac7ada0429c6f05e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 Nov 2021 13:03:57 +0800 Subject: [PATCH 02/26] [td-10564]refactor unary function. --- source/libs/function/inc/tbinoperator.h | 1 + source/libs/function/src/tbinoperator.c | 4 +++ source/libs/function/src/tscalarfunction.c | 33 +++++++++++++--------- source/libs/function/src/tunaryoperator.c | 4 +++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/source/libs/function/inc/tbinoperator.h b/source/libs/function/inc/tbinoperator.h index 65b9ce1bde..c1b6b0bc31 100644 --- a/source/libs/function/inc/tbinoperator.h +++ b/source/libs/function/inc/tbinoperator.h @@ -24,6 +24,7 @@ extern "C" { typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); +bool isBinaryStringOp(int32_t op); #ifdef __cplusplus } diff --git a/source/libs/function/src/tbinoperator.c b/source/libs/function/src/tbinoperator.c index 902c0054a0..1803e282bf 100644 --- a/source/libs/function/src/tbinoperator.c +++ b/source/libs/function/src/tbinoperator.c @@ -482,3 +482,7 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return NULL; } } + +bool isBinaryStringOp(int32_t op) { + return op == TSDB_BINARY_OP_CONCAT; +} diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 8da1a9d582..b3ffb19d7b 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -74,6 +74,10 @@ static void setScalarFuncParam(SScalarFuncParam* param, int32_t type, int32_t by param->data = pInput; } +bool isStringOp(int32_t op) { + return op == TSDB_BINARY_OP_CONCAT; +} + int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)) { if (pExprs == NULL) { @@ -87,16 +91,14 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa SScalarFuncParam leftOutput = {0}; SScalarFuncParam rightOutput = {0}; - leftOutput.data = malloc(sizeof(int64_t) * numOfRows); - if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { + leftOutput.data = malloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pLeft, numOfRows, &leftOutput, param, getSourceDataBlock); } // the right output has result from the right child syntax tree - rightOutput.data = malloc(sizeof(int64_t) * numOfRows); - - if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { + if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) { + rightOutput.data = malloc(sizeof(int64_t) * numOfRows); evaluateExprNodeTree(pRight, numOfRows, &rightOutput, param, getSourceDataBlock); } @@ -114,8 +116,6 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa } else if (pLeft->nodeType == TEXPR_VALUE_NODE) { SVariant* pVar = pRight->pVal; setScalarFuncParam(&left, pVar->nType, pVar->nLen, &pVar->i, 1); - } else { - assert(0); } if (pRight->nodeType == TEXPR_BINARYEXPR_NODE || pRight->nodeType == TEXPR_UNARYEXPR_NODE) { @@ -127,14 +127,16 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa } else if (pRight->nodeType == TEXPR_VALUE_NODE) { // exprLeft + 12 SVariant* pVar = pRight->pVal; setScalarFuncParam(&right, pVar->nType, pVar->nLen, &pVar->i, 1); - } else { - assert(0); } - OperatorFn(&left, &right, pOutput->data, TSDB_ORDER_ASC); + void* outputBuf = pOutput->data; + if (isStringOp(pExprs->_node.optr)) { + outputBuf = realloc(pOutput->data, (left.bytes + right.bytes) * left.num); + OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); + } // Set the result info - setScalarFuncParam(pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double), pOutput->data, numOfRows); + setScalarFuncParam(pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double), outputBuf, numOfRows); } else if (pExprs->nodeType == TEXPR_UNARYEXPR_NODE) { _unary_scalar_fn_t OperatorFn = getUnaryScalarOperatorFn(pExprs->_node.optr); SScalarFuncParam left = {0}; @@ -148,8 +150,13 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa } else if (pLeft->nodeType == TEXPR_VALUE_NODE) { SVariant* pVar = pLeft->pVal; setScalarFuncParam(&left, pVar->nType, pVar->nLen, &pVar->i, 1); - } else { - assert(0); + } + + // reserve enough memory buffer + if (isBinaryStringOp(pExprs->_node.optr)) { + void* outputBuf = realloc(pOutput->data, left.bytes * left.num); + assert(outputBuf != NULL); + pOutput->data = outputBuf; } OperatorFn(&left, pOutput); diff --git a/source/libs/function/src/tunaryoperator.c b/source/libs/function/src/tunaryoperator.c index dc9727d841..9105942d26 100644 --- a/source/libs/function/src/tunaryoperator.c +++ b/source/libs/function/src/tunaryoperator.c @@ -165,3 +165,7 @@ _unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { assert(0); } } + +bool isStringOperatorFn(int32_t op) { + return op == TSDB_UNARY_OP_LEN; +} From f69a885da522064d4256863dafa8d1599a793ca8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Nov 2021 13:37:31 +0800 Subject: [PATCH 03/26] [td-10564]Add implementation in executor. --- include/common/common.h | 42 +- include/common/tname.h | 4 + .../function/inc => include/common}/ttszip.h | 2 +- include/libs/function/function.h | 67 +- include/libs/parser/parser.h | 31 - include/os/os.h | 2 + include/util/tdef.h | 3 +- source/{libs/function => common}/src/ttszip.c | 0 source/libs/executor/CMakeLists.txt | 2 +- .../libs/executor/inc/executil.h | 78 +- source/libs/executor/inc/executorimpl.h | 645 ++ .../libs/executor/inc/tfilter.h | 10 +- .../libs/executor/src/executil.c | 228 +- source/libs/executor/src/executorimpl.c | 8723 +++++++++++++++++ source/libs/executor/src/tfilter.c | 3521 +++++++ source/libs/function/inc/taggfunction.h | 43 +- source/libs/function/inc/texpr.h | 1 - source/libs/function/inc/tfill.h | 25 +- source/libs/function/inc/tscalarfunction.h | 16 +- .../libs/function/inc/tscript.h | 4 +- source/libs/function/inc/tudf.h | 11 + source/libs/function/src/taggfunction.c | 272 +- source/libs/function/src/texpr.c | 2 - source/libs/function/src/tfill.c | 36 +- source/libs/function/src/tscalarfunction.c | 43 +- .../libs/function/src/tscript.c | 9 +- source/libs/function/src/tudf.c | 124 + source/libs/parser/inc/parserUtil.h | 4 +- source/libs/parser/inc/queryInfoUtil.h | 1 - source/libs/parser/src/parserUtil.c | 77 - source/libs/planner/inc/plannerInt.h | 3 +- source/libs/planner/src/planner.c | 30 +- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscSystem.c | 12 +- src/query/inc/qAggMain.h | 2 +- src/query/inc/qTableMeta.h | 4 +- src/query/src/qAggMain.c | 2 +- src/query/src/qExecutor.c | 2 +- src/query/src/qFilter.c | 4 +- 39 files changed, 13614 insertions(+), 475 deletions(-) rename {source/libs/function/inc => include/common}/ttszip.h (99%) rename source/{libs/function => common}/src/ttszip.c (100%) rename src/query/inc/qUtil.h => source/libs/executor/inc/executil.h (55%) create mode 100644 source/libs/executor/inc/executorimpl.h rename src/query/inc/qFilter.h => source/libs/executor/inc/tfilter.h (97%) rename src/query/src/qUtil.c => source/libs/executor/src/executil.c (77%) create mode 100644 source/libs/executor/src/executorimpl.c create mode 100644 source/libs/executor/src/tfilter.c rename src/query/inc/qScript.h => source/libs/function/inc/tscript.h (98%) rename src/query/src/qScript.c => source/libs/function/src/tscript.c (99%) create mode 100644 source/libs/function/src/tudf.c diff --git a/include/common/common.h b/include/common/common.h index 981614afc2..bcb48d2ce2 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -19,7 +19,7 @@ #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" - +#include "tvariant.h" //typedef struct STimeWindow { // TSKEY skey; // TSKEY ekey; @@ -66,4 +66,44 @@ typedef struct SColumnInfoData { char *pData; // the corresponding block data in memory } SColumnInfoData; +//====================================================================================================================== +// the following structure shared by parser and executor +typedef struct SLimit { + int64_t limit; + int64_t offset; +} SLimit; + +typedef struct SOrder { + uint32_t order; + int32_t orderColId; +} SOrder; + +typedef struct SGroupbyExpr { + int16_t tableIndex; + SArray* columnInfo; // SArray, group by columns information + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SGroupbyExpr; + +// the structure for sql function in select clause +typedef struct SSqlExpr { + char token[TSDB_COL_NAME_LEN]; // original token + SSchema resSchema; + SColIndex colInfo; // there may be mutiple input columns + uint64_t uid; // table uid, todo refactor use the pointer + int32_t interBytes; // inter result buffer size + int16_t numOfParams; // argument value of each function + SVariant param[3]; // parameters are not more than 3 +} SSqlExpr; + +typedef struct SExprInfo { + struct SSqlExpr base; + struct tExprNode *pExpr; +} SExprInfo; + +#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_DESC_FORWARD_STEP -1 + +#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) + #endif // TDENGINE_COMMON_H diff --git a/include/common/tname.h b/include/common/tname.h index 18526f54d4..7578a7804c 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H +#include "taosmsg.h" + #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 @@ -52,6 +54,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type); int32_t tNameSetAcctId(SName* dst, const char* acct); +SSchema* tGetTbnameColumnSchema(); + #if 0 int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); #endif diff --git a/source/libs/function/inc/ttszip.h b/include/common/ttszip.h similarity index 99% rename from source/libs/function/inc/ttszip.h rename to include/common/ttszip.h index 95be18f4ea..38699ae791 100644 --- a/source/libs/function/inc/ttszip.h +++ b/include/common/ttszip.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -#include "taosdef.h" +#include "tdef.h" #include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 1d1480606e..b91cc83255 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -78,13 +78,28 @@ extern "C" { #define FUNCTION_MODE 36 #define FUNCTION_SAMPLE 37 +// determine the real data need to calculated the result +enum { + BLK_DATA_NO_NEEDED = 0x0, + BLK_DATA_STATIS_NEEDED = 0x1, + BLK_DATA_ALL_NEEDED = 0x3, + BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter +}; + +enum { + MASTER_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, + REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, +}; + typedef struct SPoint1 { int64_t key; union{double val; char* ptr;}; } SPoint1; struct SQLFunctionCtx; -struct SResultRowCellInfo; +struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SExtTagsInfo { @@ -93,6 +108,8 @@ typedef struct SExtTagsInfo { struct SQLFunctionCtx **pTagCtxList; } SExtTagsInfo; +#define GET_RES_INFO(ctx) ((ctx)->resultInfo) + // sql function runtime context typedef struct SQLFunctionCtx { int32_t size; // number of rows @@ -117,9 +134,9 @@ typedef struct SQLFunctionCtx { void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SVariant tag; - bool isSmaSet; - SColumnDataAgg sma; - struct SResultRowCellInfo *resultInfo; + bool isAggSet; + SColumnDataAgg agg; + struct SResultRowEntryInfo *resultInfo; SExtTagsInfo tagInfo; SPoint1 start; SPoint1 end; @@ -161,7 +178,7 @@ typedef struct SAggFunctionInfo { int8_t sFunctionId; // Transfer function for super table query uint16_t status; - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment + bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment void (*exec)(SQLFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. @@ -176,7 +193,7 @@ typedef struct SScalarFunctionInfo { int8_t type; // scalar function or aggregation function uint8_t functionId; // index of scalar function - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment + bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment void (*exec)(SQLFunctionCtx *pCtx); } SScalarFunctionInfo; @@ -221,10 +238,48 @@ bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* funct const char* qGetFunctionName(int32_t functionId); +tExprNode* exprTreeFromBinary(const void* data, size_t size); + void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc); tExprNode* exprdup(tExprNode* pTree); +void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num); +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); +int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num); +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); + +struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num); +void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num); +struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index); + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// fill api +struct SFillInfo; +struct SFillColInfo; + +typedef struct SPoint { + int64_t key; + void * val; +} SPoint; + +void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); +void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); +void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal); +bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); + +struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, + int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, + struct SFillColInfo* pFillCol, void* handle); + +void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); +int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); +int64_t getFillInfoStart(struct SFillInfo *pFillInfo); + +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); + #ifdef __cplusplus } #endif diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index d9a7b0ea41..ff28a03260 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -32,21 +32,7 @@ typedef struct SColumn { SColumnInfo info; } SColumn; -// the structure for sql function in select clause -typedef struct SSqlExpr { - char token[TSDB_COL_NAME_LEN]; // original token - SSchema resSchema; - SColIndex colInfo; - uint64_t uid; // table uid, todo refactor use the pointer - int32_t interBytes; // inter result buffer size - int16_t numOfParams; // argument value of each function - SVariant param[3]; // parameters are not more than 3 -} SSqlExpr; -typedef struct SExprInfo { - SSqlExpr base; - struct tExprNode *pExpr; -} SExprInfo; //typedef struct SInterval { // int32_t tz; // query client timezone @@ -63,13 +49,6 @@ typedef struct SExprInfo { // int32_t primaryColId; // primary timestamp column //} SSessionWindow; -typedef struct SGroupbyExpr { - int16_t tableIndex; - SArray* columnInfo; // SArray, group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SGroupbyExpr; - typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; @@ -82,16 +61,6 @@ typedef struct SFieldInfo { SArray *internalField; // SArray } SFieldInfo; -typedef struct SLimit { - int64_t limit; - int64_t offset; -} SLimit; - -typedef struct SOrder { - uint32_t order; - int32_t orderColId; -} SOrder; - typedef struct SCond { uint64_t uid; int32_t len; // length of tag query condition data diff --git a/include/os/os.h b/include/os/os.h index ac36611a1b..44ce6f81ec 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -45,6 +45,8 @@ extern "C" { #include #include #include +#include +#include #include "osAtomic.h" #include "osDef.h" diff --git a/include/util/tdef.h b/include/util/tdef.h index 7a684ba2e1..fca9a1395b 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -140,7 +140,8 @@ do { \ #define TSDB_UNARY_OP_ROUND 4503 #define TSDB_UNARY_OP_LEN 4600 - +#define TSDB_UNARY_OP_LTRIM 4601 +#define TSDB_UNARY_OP_RTRIM 4601 #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) diff --git a/source/libs/function/src/ttszip.c b/source/common/src/ttszip.c similarity index 100% rename from source/libs/function/src/ttszip.c rename to source/common/src/ttszip.c diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 9f02f4a4c7..a6f70b9e83 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -8,5 +8,5 @@ target_include_directories( target_link_libraries( executor - PRIVATE os util common + PRIVATE os util common function parser ) \ No newline at end of file diff --git a/src/query/inc/qUtil.h b/source/libs/executor/inc/executil.h similarity index 55% rename from src/query/inc/qUtil.h rename to source/libs/executor/inc/executil.h index ce607f0fe2..7e910d5674 100644 --- a/src/query/inc/qUtil.h +++ b/source/libs/executor/inc/executil.h @@ -15,6 +15,8 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "common.h" +#include "tpagedfile.h" #include "tbuffer.h" #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ @@ -40,42 +42,92 @@ #define curTimeWindowIndex(_winres) ((_winres)->curIndex) -int32_t getOutputInterResultBufSize(SQueryAttr* pQueryAttr); +struct SColumnFilterElem; -size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv); +typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type); + +typedef struct SGroupResInfo { + int32_t totalGroup; + int32_t currentGroup; + int32_t index; + SArray* pRows; // SArray + bool ordered; + int32_t position; +} SGroupResInfo; + +typedef struct SResultRow { + int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer + int32_t offset:29; // row index in buffer page + bool startInterp; // the time window start timestamp has done the interpolation already. + bool endInterp; // the time window end timestamp has done the interpolation already. + bool closed; // this result status: closed or opened + uint32_t numOfRows; // number of rows of current time window + struct SResultRowEntryInfo* pEntryInfo; // For each result column, there is a resultInfo + STimeWindow win; + char *key; // start key of current result row +} SResultRow; + +typedef struct SResultRowInfo { + SResultRow** pResult; // result list + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t capacity; // max capacity + int32_t curPos; // current active result row index of pResult list +} SResultRowInfo; + +typedef struct SResultRowPool { + int32_t elemSize; + int32_t blockSize; + int32_t numOfElemPerBlock; + + struct { + int32_t blockIndex; + int32_t pos; + } position; + + SArray* pData; // SArray +} SResultRowPool; + +struct SQueryAttr; +struct SQueryRuntimeEnv; +struct SUdfInfo; + +int32_t getOutputInterResultBufSize(struct SQueryAttr* pQueryAttr); + +size_t getResultRowSize(struct SQueryRuntimeEnv* pRuntimeEnv); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void resetResultRowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); +void resetResultRowInfo(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); +void clearResultRow(struct SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); +struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr); void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols); -int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); +int32_t getRowNumForMultioutput(struct SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char* getPosInResultPage(SQueryAttr* pQueryAttr, tFilePage* page, int32_t rowOffset, +static FORCE_INLINE char* getPosInResultPage(struct SQueryAttr* pQueryAttr, SFilePage* page, int32_t rowOffset, int32_t offset) { assert(rowOffset >= 0 && pQueryAttr != NULL); - int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); - return ((char *)page->data) + rowOffset + offset * numOfRows; +// int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); +// return ((char *)page->data) + rowOffset + offset * numOfRows; } -bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); -bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); +//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); +//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); __filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr); @@ -103,8 +155,8 @@ bool hasRemainData(SGroupResInfo* pGroupResInfo); bool incNextGroup(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); -int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); -int32_t initUdfInfo(SUdfInfo* pUdfInfo); +int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h new file mode 100644 index 0000000000..a99717a123 --- /dev/null +++ b/source/libs/executor/inc/executorimpl.h @@ -0,0 +1,645 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_EXECUTORIMPL_H +#define TDENGINE_EXECUTORIMPL_H + +#include "os.h" +#include "common.h" +#include "ttszip.h" +#include "tvariant.h" + +#include "thash.h" +//#include "parser.h" +#include "executil.h" +#include "taosdef.h" +#include "tarray.h" +#include "tfilter.h" +#include "tlockfree.h" +#include "tpagedfile.h" + +struct SColumnFilterElem; + +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + +typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); + +#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) +#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) + +#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) + +#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) + +#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) + +enum { + // when query starts to execute, this status will set + QUERY_NOT_COMPLETED = 0x1u, + + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed + */ + QUERY_COMPLETED = 0x2u, + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option + */ + QUERY_OVER = 0x4u, +}; + + +typedef struct SResultRowCell { + uint64_t groupId; + SResultRow *pRow; +} SResultRowCell; + +/** + * If the number of generated results is greater than this value, + * query query will be halt and return results to client immediate. + */ +typedef struct SRspResultInfo { + int64_t total; // total generated result size in rows + int32_t capacity; // capacity of current result output buffer + int32_t threshold; // result size threshold in rows. +} SRspResultInfo; + +typedef struct SColumnFilterElem { + int16_t bytes; // column length + __filter_func_t fp; + SColumnFilterInfo filterInfo; + void *q; +} SColumnFilterElem; + +typedef struct SSingleColumnFilterInfo { + void* pData; + void* pData2; //used for nchar column + int32_t numOfFilters; + SColumnInfo info; + SColumnFilterElem* pFilters; +} SSingleColumnFilterInfo; + +typedef struct STableQueryInfo { + TSKEY lastKey; + int32_t groupIndex; // group id in table list + SVariant tag; + STimeWindow win; + STSCursor cur; + void* pTable; // for retrieve the page id list + SResultRowInfo resInfo; +} STableQueryInfo; + +typedef enum { + QUERY_PROF_BEFORE_OPERATOR_EXEC = 0, + QUERY_PROF_AFTER_OPERATOR_EXEC, + QUERY_PROF_QUERY_ABORT +} EQueryProfEventType; + +typedef struct { + EQueryProfEventType eventType; + int64_t eventTime; + + union { + uint8_t operatorType; //for operator event + int32_t abortCode; //for query abort event + }; +} SQueryProfEvent; + +typedef struct { + uint8_t operatorType; + int64_t sumSelfTime; + int64_t sumRunTimes; +} SOperatorProfResult; + +typedef struct SQueryCostInfo { + uint64_t loadStatisTime; + uint64_t loadFileBlockTime; + uint64_t loadDataInCacheTime; + uint64_t loadStatisSize; + uint64_t loadFileBlockSize; + uint64_t loadDataInCacheSize; + + uint64_t loadDataTime; + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t discardBlocks; + uint64_t elapsedTime; + uint64_t firstStageMergeTime; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; + uint64_t numOfTimeWindows; + + SArray* queryProfEvents; //SArray + SHashObj* operatorProfResults; //map +} SQueryCostInfo; + +typedef struct { + int64_t vgroupLimit; + int64_t ts; +} SOrderedPrjQueryInfo; + +typedef struct { + char* tags; + SArray* pResult; // SArray +} SInterResult; + +// The basic query information extracted from the SQueryInfo tree to support the +// execution of query in a data node. +typedef struct SQueryAttr { + SLimit limit; + SLimit slimit; + + // todo comment it + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo;// if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + bool tsCompQuery; // is tscomp query + bool diffQuery; // is diff query + bool simpleAgg; + bool pointInterpQuery; // point interpolation query + bool needReverseScan; // need reverse scan + bool distinct; // distinct query or not + bool stateWindow; // window State on sub/normal table + bool createFilterOperator; // if filter operator is needed + bool multigroupResult; // multigroup result can exist in one SSDataBlock + int32_t interBufSize; // intermediate buffer sizse + + int32_t havingNum; // having expr number + + SOrder order; + int16_t numOfCols; + int16_t numOfTags; + + STimeWindow window; + SInterval interval; + SSessionWindow sw; + int16_t precision; + int16_t numOfOutput; + int16_t fillType; + + int32_t srcRowSize; // todo extract struct + int32_t resultRowSize; + int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. + int32_t maxTableColumnWidth; + int32_t tagLen; // tag value length of current query + SGroupbyExpr *pGroupbyExpr; + + SExprInfo* pExpr1; + SExprInfo* pExpr2; + int32_t numOfExpr2; + SExprInfo* pExpr3; + int32_t numOfExpr3; + + SColumnInfo* tableCols; + SColumnInfo* tagColList; + int32_t numOfFilterCols; + int64_t* fillVal; + SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. + + SSingleColumnFilterInfo* pFilterInfo; +// SFilterInfo *pFilters; + + void* tsdb; +// SMemRef memRef; + STableGroupInfo tableGroupInfo; // table list SArray + int32_t vgId; + SArray *pUdfInfo; // no need to free +} SQueryAttr; + +typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); +typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); + +struct SOperatorInfo; + +typedef struct SQueryRuntimeEnv { + jmp_buf env; + SQueryAttr* pQueryAttr; + uint32_t status; // query status + void* qinfo; + uint8_t scanFlag; // denotes reversed scan of data or not + void* pQueryHandle; + + int32_t prevGroupId; // previous executed group id + bool enableGroupData; + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + char** prevRow; + + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; + + char* tagVal; // tag value of current data block + struct SScalarFunctionSupport * scalarSup; + + SSDataBlock *outputBuf; + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + struct SOperatorInfo *proot; + SGroupResInfo groupResInfo; + int64_t currentOffset; // dynamic offset value + + STableQueryInfo *current; + SRspResultInfo resultInfo; + SHashObj *pTableRetrieveTsMap; + struct SUdfInfo *pUdfInfo; +} SQueryRuntimeEnv; + +enum { + OP_IN_EXECUTING = 1, + OP_RES_TO_RETURN = 2, + OP_EXEC_DONE = 3, +}; + +enum OPERATOR_TYPE_E { + OP_TableScan = 1, + OP_DataBlocksOptScan = 2, + OP_TableSeqScan = 3, + OP_TagScan = 4, + OP_TableBlockInfoScan= 5, + OP_Aggregate = 6, + OP_Project = 7, + OP_Groupby = 8, + OP_Limit = 9, + OP_SLimit = 10, + OP_TimeWindow = 11, + OP_SessionWindow = 12, + OP_Fill = 13, + OP_MultiTableAggregate = 14, + OP_MultiTableTimeInterval = 15, + OP_DummyInput = 16, //TODO remove it after fully refactor. + OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. + OP_GlobalAggregate = 18, // global merge for the multi-way data sources. + OP_Filter = 19, + OP_Distinct = 20, + OP_Join = 21, + OP_StateWindow = 22, + OP_AllTimeWindow = 23, + OP_AllMultiTableTimeInterval = 24, + OP_Order = 25, +}; + +typedef struct SOperatorInfo { + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char *name; // name, used to show the query execution plan + void *info; // extension attribution + SExprInfo *pExpr; + SQueryRuntimeEnv *pRuntimeEnv; + + struct SOperatorInfo **upstream; // upstream pointer list + int32_t numOfUpstream; // number of upstream. The value is always ONE expect for join operator + __operator_fn_t exec; + __optr_cleanup_fn_t cleanup; +} SOperatorInfo; + +enum { + QUERY_RESULT_NOT_READY = 1, + QUERY_RESULT_READY = 2, +}; + +typedef struct { + int32_t numOfTags; + int32_t numOfCols; + SColumnInfo *colList; +} SQueriedTableInfo; + +typedef struct SQInfo { + void* signature; + uint64_t qId; + int32_t code; // error code to returned to client + int64_t owner; // if it is in execution + + SQueryRuntimeEnv runtimeEnv; + SQueryAttr query; + void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + + pthread_mutex_t lock; // used to synchronize the rsp/query threads + tsem_t ready; + int32_t dataReady; // denote if query result is ready or not + void* rspContext; // response context + int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string + SQueryCostInfo summary; +} SQInfo; + +typedef struct SQueryParam { + char *sql; + char *tagCond; + char *colCond; + char *tbnameCond; + char *prevResult; + SArray *pTableIdList; + SSqlExpr **pExpr; + SSqlExpr **pSecExpr; + SExprInfo *pExprs; + SExprInfo *pSecExprs; + + SFilterInfo *pFilters; + + SColIndex *pGroupColIndex; + SColumnInfo *pTagColumnInfo; + SGroupbyExpr *pGroupbyExpr; + int32_t tableScanOperator; + SArray *pOperator; + struct SUdfInfo *pUdfInfo; +} SQueryParam; + +typedef struct STableScanInfo { + void *pQueryHandle; + int32_t numOfBlocks; + int32_t numOfSkipped; + int32_t numOfBlockStatis; + int64_t numOfRows; + + int32_t order; // scan order + int32_t times; // repeat counts + int32_t current; + int32_t reverseTimes; // 0 by default + + SQLFunctionCtx *pCtx; // next operator query context + SResultRowInfo *pResultRowInfo; + int32_t *rowCellInfoOffset; + SExprInfo *pExpr; + SSDataBlock block; + int32_t numOfOutput; + int64_t elapsedTime; + + int32_t tableIndex; + int32_t prevGroupId; // previous table group id +} STableScanInfo; + +typedef struct STagScanInfo { + SColumnInfo* pCols; + SSDataBlock* pRes; + int32_t totalTables; + int32_t curPos; +} STagScanInfo; + +typedef struct SOptrBasicInfo { + SResultRowInfo resultRowInfo; + int32_t *rowCellInfoOffset; // offset value for each row result cell info + SQLFunctionCtx *pCtx; + SSDataBlock *pRes; +} SOptrBasicInfo; + +typedef struct SOptrBasicInfo STableIntervalOperatorInfo; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + uint32_t seed; +} SAggOperatorInfo; + +typedef struct SProjectOperatorInfo { + SOptrBasicInfo binfo; + int32_t bufCapacity; + uint32_t seed; + + SSDataBlock *existDataBlock; +} SProjectOperatorInfo; + +typedef struct SLimitOperatorInfo { + int64_t limit; + int64_t total; +} SLimitOperatorInfo; + +typedef struct SSLimitOperatorInfo { + int64_t groupTotal; + int64_t currentGroupOffset; + + int64_t rowsTotal; + int64_t currentOffset; + SLimit limit; + SLimit slimit; + + char **prevRow; + SArray *orderColumnList; + bool hasPrev; + bool ignoreCurrentGroup; + bool multigroupResult; + SSDataBlock *pRes; // result buffer + SSDataBlock *pPrevBlock; + int64_t capacity; + int64_t threshold; +} SSLimitOperatorInfo; + +typedef struct SFilterOperatorInfo { + SSingleColumnFilterInfo *pFilterInfo; + int32_t numOfFilterCols; +} SFilterOperatorInfo; + +typedef struct SFillOperatorInfo { + struct SFillInfo *pFillInfo; + SSDataBlock *pRes; + int64_t totalInputRows; + void **p; + SSDataBlock *existNewGroupBlock; + bool multigroupResult; +} SFillOperatorInfo; + +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + int32_t colIndex; + char *prevData; // previous group by value +} SGroupbyOperatorInfo; + +typedef struct SSWindowOperatorInfo { + SOptrBasicInfo binfo; + STimeWindow curWindow; // current time window + TSKEY prevTs; // previous timestamp + int32_t numOfRows; // number of rows + int32_t start; // start row index + bool reptScan; // next round scan +} SSWindowOperatorInfo; + +typedef struct SStateWindowOperatorInfo { + SOptrBasicInfo binfo; + STimeWindow curWindow; // current time window + int32_t numOfRows; // number of rows + int32_t colIndex; // start row index + int32_t start; + char* prevData; // previous data + bool reptScan; +} SStateWindowOperatorInfo; + +typedef struct SDistinctDataInfo { + int32_t index; + int32_t type; + int32_t bytes; +} SDistinctDataInfo; + +typedef struct SDistinctOperatorInfo { + SHashObj *pSet; + SSDataBlock *pRes; + bool recordNullVal; //has already record the null value, no need to try again + int64_t threshold; + int64_t outputCapacity; + int32_t totalBytes; + char* buf; + SArray* pDistinctDataInfo; +} SDistinctOperatorInfo; + +struct SGlobalMerger; + +typedef struct SMultiwayMergeInfo { + struct SGlobalMerger *pMerge; + SOptrBasicInfo binfo; + int32_t bufCapacity; + int64_t seed; + char **prevRow; + SArray *orderColumnList; + int32_t resultRowFactor; + + bool hasGroupColData; + char **currentGroupColData; + SArray *groupColumnList; + bool hasDataBlockForNewGroup; + SSDataBlock *pExistBlock; + + SArray *udfInfo; + bool hasPrev; + bool multiGroupResults; +} SMultiwayMergeInfo; + +// todo support the disk-based sort +typedef struct SOrderOperatorInfo { + int32_t colIndex; + int32_t order; + SSDataBlock *pDataBlock; +} SOrderOperatorInfo; + +void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); + +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); + +SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); +SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, + int32_t numOfRows, void* merger); +SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); +SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); +SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); + +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); +SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); + +SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +SSDataBlock* doSLimit(void* param, bool* newgroup); + +int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); +bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); +void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); + +SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); + +void* destroyOutputBuf(SSDataBlock* pBlock); +void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); + +void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); +void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); + +void freeParam(SQueryParam *param); +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); +int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); + +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, + SSqlExpr **pExpr, SExprInfo *prevExpr, struct SUdfInfo *pUdfInfo); + +int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); + +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo); + +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, + int32_t prevResultLen, void* merger); + +int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId); +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); + +STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); +STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); + +int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); + +bool isQueryKilled(SQInfo *pQInfo); +int32_t checkForQueryBuf(size_t numOfTables); +bool checkNeedToCompressQueryCol(SQInfo *pQInfo); +bool doBuildResCheck(SQInfo* pQInfo); +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); + +bool onlyQueryTags(SQueryAttr* pQueryAttr); +void destroyUdfInfo(struct SUdfInfo* pUdfInfo); + +bool isValidQInfo(void *param); + +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen); + +size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); +void setQueryKilled(SQInfo *pQInfo); + +void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); +void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code); +void calculateOperatorProfResults(SQInfo* pQInfo); +void queryCostStatis(SQInfo *pQInfo); + +void freeQInfo(SQInfo *pQInfo); +void freeQueryAttr(SQueryAttr *pQuery); + +int32_t getMaximumIdleDurationSec(); + +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); + +#endif // TDENGINE_EXECUTORIMPL_H diff --git a/src/query/inc/qFilter.h b/source/libs/executor/inc/tfilter.h similarity index 97% rename from src/query/inc/qFilter.h rename to source/libs/executor/inc/tfilter.h index af45b816f9..55edf27949 100644 --- a/src/query/inc/qFilter.h +++ b/source/libs/executor/inc/tfilter.h @@ -20,9 +20,9 @@ extern "C" { #endif -#include "texpr.h" -#include "hash.h" +#include "thash.h" #include "tname.h" +#include "function.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -105,7 +105,7 @@ typedef struct SFilterColRange { typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t); typedef int32_t(*filter_desc_compare_func)(const void *, const void *); -typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t); +typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t); typedef struct SFilterRangeCompare { int64_t s; @@ -324,13 +324,13 @@ typedef struct SFilterInfo { extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options); -extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols); +extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SDataStatis *pDataStatis, int32_t numOfCols, int32_t numOfRows); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus } diff --git a/src/query/src/qUtil.c b/source/libs/executor/src/executil.c similarity index 77% rename from src/query/src/qUtil.c rename to source/libs/executor/src/executil.c index f52d6de5d2..1f7795e06f 100644 --- a/src/query/src/qUtil.c +++ b/source/libs/executor/src/executil.c @@ -15,11 +15,11 @@ #include "os.h" #include "taosmsg.h" -#include "hash.h" +#include "thash.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "queryLog.h" +#include "executil.h" +#include "executorimpl.h" +//#include "queryLog.h" #include "tbuffer.h" #include "tcompression.h" #include "tlosertree.h" @@ -33,9 +33,9 @@ typedef struct SCompSupporter { int32_t getRowNumForMultioutput(SQueryAttr* pQueryAttr, bool topBottomQuery, bool stable) { if (pQueryAttr && (!stable)) { for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - if (pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_TOP || pQueryAttr->pExpr1[i].base.functionId == TSDB_FUNC_BOTTOM) { - return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i64; - } +// if (pQueryAttr->pExpr1[i].base. == FUNCTION_TOP || pQueryAttr->pExpr1[i].base.functionId == FUNCTION_BOTTOM) { +// return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i; +// } } } @@ -143,18 +143,18 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 // the result does not put into the SDiskbasedResultBuf, ignore it. if (pResultRow->pageId >= 0) { - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); + SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); int16_t offset = 0; for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) { - SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; + struct SResultRowEntryInfo *pEntryInfo = NULL;//pResultRow->pEntryInfo[i]; - int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resType; + int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resSchema.bytes; char * s = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResultRow->offset, offset); memset(s, 0, size); offset += size; - RESET_RESULT_INFO(pResultInfo); + cleanupResultRowEntry(pEntryInfo); } } @@ -168,14 +168,16 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 } // TODO refactor: use macro -SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { +struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { assert(index >= 0 && offset != NULL); - return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[index]); +// return (SResultRowEntryInfo*)((char*) pRow->pCellInfo + offset[index]); +return NULL; } size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - return (pQueryAttr->numOfOutput * sizeof(SResultRowCellInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); + return 0; +// return (pQueryAttr->numOfOutput * sizeof(SResultRowEntryInfo)) + pQueryAttr->interBufSize + sizeof(SResultRow); } SResultRowPool* initResultRowPool(size_t size) { @@ -271,9 +273,9 @@ void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen) { tbufWriteUint32(bw, numOfRows); for(int32_t k = 0; k < numOfRows; ++k) { - SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); - tbufWriteDouble(bw, v.avg); - tbufWriteInt64(bw, v.key); +// SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); +// tbufWriteDouble(bw, v.avg); +// tbufWriteInt64(bw, v.key); } } } @@ -301,19 +303,19 @@ SArray* interResFromBinary(const char* data, int32_t len) { SArray* p = taosArrayInit(numOfCols, sizeof(SStddevInterResult)); for(int32_t j = 0; j < numOfCols; ++j) { - int16_t colId = tbufReadUint16(&br); +// int16_t colId = tbufReadUint16(&br); int32_t numOfRows = tbufReadUint32(&br); - SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; +// SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; for(int32_t k = 0; k < numOfRows; ++k) { - SResPair px = {0}; - px.avg = tbufReadDouble(&br); - px.key = tbufReadInt64(&br); - - taosArrayPush(interRes.pResult, &px); +// SResPair px = {0}; +// px.avg = tbufReadDouble(&br); +// px.key = tbufReadInt64(&br); +// +// taosArrayPush(interRes.pResult, &px); } - taosArrayPush(p, &interRes); +// taosArrayPush(p, &interRes); } char* p1 = NULL; @@ -395,22 +397,22 @@ static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { - int32_t functionId = pQueryAttr->pExpr1[j].base.functionId; + int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; /* * ts, tag, tagprj function can not decide the output number of current query * the number of output result is decided by main output */ - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { continue; } - SResultRowCellInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); - assert(pResultInfo != NULL); - - if (pResultInfo->numOfRes > 0) { - return pResultInfo->numOfRes; - } +// SResultRowEntryInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); +// assert(pResultInfo != NULL); +// +// if (pResultInfo->numOfRes > 0) { +// return pResultInfo->numOfRes; +// } } return 0; @@ -545,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEn pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { - qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); +// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -617,8 +619,8 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEn int64_t endt = taosGetTimestampMs(); - qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), - pGroupResInfo->currentGroup, endt - startt); +// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), +// pGroupResInfo->currentGroup, endt - startt); _end: tfree(pTableQueryInfoList); @@ -639,90 +641,90 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRu break; } - qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); +// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } - int64_t elapsedTime = taosGetTimestampUs() - st; - qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), - pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); +// int64_t elapsedTime = taosGetTimestampUs() - st; +// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), +// pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); return TSDB_CODE_SUCCESS; } -void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { - tbufWriteUint32(bw, pDist->numOfTables); - tbufWriteUint16(bw, pDist->numOfFiles); - tbufWriteUint64(bw, pDist->totalSize); - tbufWriteUint64(bw, pDist->totalRows); - tbufWriteInt32(bw, pDist->maxRows); - tbufWriteInt32(bw, pDist->minRows); - tbufWriteUint32(bw, pDist->numOfRowsInMemTable); - tbufWriteUint32(bw, pDist->numOfSmallBlocks); - tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); +//void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { +// tbufWriteUint32(bw, pDist->numOfTables); +// tbufWriteUint16(bw, pDist->numOfFiles); +// tbufWriteUint64(bw, pDist->totalSize); +// tbufWriteUint64(bw, pDist->totalRows); +// tbufWriteInt32(bw, pDist->maxRows); +// tbufWriteInt32(bw, pDist->minRows); +// tbufWriteUint32(bw, pDist->numOfRowsInMemTable); +// tbufWriteUint32(bw, pDist->numOfSmallBlocks); +// tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); +// +// // compress the binary string +// char* p = TARRAY_GET_START(pDist->dataBlockInfos); +// +// // compress extra bytes +// size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; +// char* tmp = malloc(x + 2); +// +// bool comp = false; +// int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); +// if (len == -1 || len >= x) { // compress failed, do not compress this binary data +// comp = false; +// len = (int32_t)x; +// } else { +// comp = true; +// } +// +// tbufWriteUint8(bw, comp); +// tbufWriteUint32(bw, len); +// if (comp) { +// tbufWriteBinary(bw, tmp, len); +// } else { +// tbufWriteBinary(bw, p, len); +// } +// tfree(tmp); +//} - // compress the binary string - char* p = TARRAY_GET_START(pDist->dataBlockInfos); - - // compress extra bytes - size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; - char* tmp = malloc(x + 2); - - bool comp = false; - int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); - if (len == -1 || len >= x) { // compress failed, do not compress this binary data - comp = false; - len = (int32_t)x; - } else { - comp = true; - } - - tbufWriteUint8(bw, comp); - tbufWriteUint32(bw, len); - if (comp) { - tbufWriteBinary(bw, tmp, len); - } else { - tbufWriteBinary(bw, p, len); - } - tfree(tmp); -} - -void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { - SBufferReader br = tbufInitReader(data, len, false); - - pDist->numOfTables = tbufReadUint32(&br); - pDist->numOfFiles = tbufReadUint16(&br); - pDist->totalSize = tbufReadUint64(&br); - pDist->totalRows = tbufReadUint64(&br); - pDist->maxRows = tbufReadInt32(&br); - pDist->minRows = tbufReadInt32(&br); - pDist->numOfRowsInMemTable = tbufReadUint32(&br); - pDist->numOfSmallBlocks = tbufReadUint32(&br); - int64_t numSteps = tbufReadUint64(&br); - - bool comp = tbufReadUint8(&br); - uint32_t compLen = tbufReadUint32(&br); - - size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); - - char* outputBuf = NULL; - if (comp) { - outputBuf = malloc(originalLen); - - size_t actualLen = compLen; - const char* compStr = tbufReadBinary(&br, &actualLen); - - int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, - (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); - } else { - outputBuf = (char*) tbufReadBinary(&br, &originalLen); - } - - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); - if (comp) { - tfree(outputBuf); - } -} +//void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { +// SBufferReader br = tbufInitReader(data, len, false); +// +// pDist->numOfTables = tbufReadUint32(&br); +// pDist->numOfFiles = tbufReadUint16(&br); +// pDist->totalSize = tbufReadUint64(&br); +// pDist->totalRows = tbufReadUint64(&br); +// pDist->maxRows = tbufReadInt32(&br); +// pDist->minRows = tbufReadInt32(&br); +// pDist->numOfRowsInMemTable = tbufReadUint32(&br); +// pDist->numOfSmallBlocks = tbufReadUint32(&br); +// int64_t numSteps = tbufReadUint64(&br); +// +// bool comp = tbufReadUint8(&br); +// uint32_t compLen = tbufReadUint32(&br); +// +// size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); +// +// char* outputBuf = NULL; +// if (comp) { +// outputBuf = malloc(originalLen); +// +// size_t actualLen = compLen; +// const char* compStr = tbufReadBinary(&br, &actualLen); +// +// int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, +// (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); +// assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); +// } else { +// outputBuf = (char*) tbufReadBinary(&br, &originalLen); +// } +// +// pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); +// if (comp) { +// tfree(outputBuf); +// } +//} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c new file mode 100644 index 0000000000..70b3c8982f --- /dev/null +++ b/source/libs/executor/src/executorimpl.c @@ -0,0 +1,8723 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include "ttime.h" +#include "taosmsg.h" +#include "tglobal.h" + +#include "exception.h" +#include "executorimpl.h" +#include "thash.h" +//#include "queryLog.h" +#include "function.h" +#include "tcompare.h" +#include "tcompression.h" +#include "tlosertree.h" +#include "ttypes.h" + +#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) +#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) +#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN) +#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) +#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) + +#define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) +#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) + +#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} + +#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) + +#define MULTI_KEY_DELIM "-" + +#define TIME_WINDOW_COPY(_dst, _src) do {\ + (_dst).skey = (_src).skey;\ + (_dst).ekey = (_src).ekey;\ +} while (0) + +enum { + TS_JOIN_TS_EQUAL = 0, + TS_JOIN_TS_NOT_EQUALS = 1, + TS_JOIN_TAG_NOT_EQUALS = 2, +}; + +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; + +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 1000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 1000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + +#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) +#define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) +#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) + +#define TSKEY_MAX_ADD(a,b) \ +do { \ + if (a < 0) { a = a + b; break;} \ + if (sizeof(a) == sizeof(int32_t)) { \ + if((b) > 0 && ((b) >= INT32_MAX - (a))){\ + a = INT32_MAX; \ + } else { \ + a = a + b; \ + } \ + } else { \ + if((b) > 0 && ((b) >= INT64_MAX - (a))){\ + a = INT64_MAX; \ + } else { \ + a = a + b; \ + } \ + } \ +} while(0) + +#define TSKEY_MIN_SUB(a,b) \ +do { \ + if (a >= 0) { a = a + b; break;} \ + if (sizeof(a) == sizeof(int32_t)){ \ + if((b) < 0 && ((b) <= INT32_MIN - (a))){\ + a = INT32_MIN; \ + } else { \ + a = a + b; \ + } \ + } else { \ + if((b) < 0 && ((b) <= INT64_MIN-(a))) {\ + a = INT64_MIN; \ + } else { \ + a = a + b; \ + } \ + } \ +} while (0) + +uint64_t queryHandleId = 0; + +int32_t getMaximumIdleDurationSec() { + return tsShellActivityTimer * 2; +} +int64_t genQueryId(void) { + int64_t uid = 0; + int64_t did = 0;//tsDnodeId; + + uid = did << 54; + + int64_t pid = ((int64_t)taosGetPId()) & 0x3FF; + + uid |= pid << 44; + + int64_t ts = taosGetTimestampMs() & 0x1FFFFFFFF; + + uid |= ts << 11; + + int64_t sid = atomic_add_fetch_64(&queryHandleId, 1) & 0x7FF; + + uid |= sid; + +// //qDebug("gen qid:0x%"PRIx64, uid); + + return uid; +} + +static int32_t getExprFunctionId(SExprInfo *pExprInfo) { + assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); + return pExprInfo->pExpr->_node.functionId; +} + +static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { + tw->skey += pQueryAttr->interval.sliding * factor; + tw->ekey = tw->skey + pQueryAttr->interval.interval - 1; + return; + } + + int64_t key = tw->skey, interval = pQueryAttr->interval.interval; + //convert key to second + key = convertTimePrecision(key, pQueryAttr->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + + if (pQueryAttr->interval.intervalUnit == 'y') { + interval *= 12; + } + + struct tm tm; + time_t t = (time_t)key; + localtime_r(&t, &tm); + + int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + + mon = (int)(mon + interval); + tm.tm_year = mon / 12; + tm.tm_mon = mon % 12; + tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + + tw->ekey -= 1; +} + +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); +static void setResultOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset); + +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); + +static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); + +static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); +static bool hasMainOutput(SQueryAttr *pQueryAttr); + +static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols); + +static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo); +static void releaseQueryBuf(size_t numOfTables); +static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); +//static STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win); +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); + +static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); + +static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); +static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); +static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOperatorInfo(SOperatorInfo* pOperator); + +static void doSetOperatorCompleted(SOperatorInfo* pOperator) { + pOperator->status = OP_EXEC_DONE; + if (pOperator->pRuntimeEnv != NULL) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + } +} + +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); + +static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); + +static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo); +static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, + SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); + +SArray* getOrderCheckColumns(SQueryAttr* pQuery); + + +typedef struct SRowCompSupporter { + SQueryRuntimeEnv *pRuntimeEnv; + int16_t dataOffset; + __compar_fn_t comFunc; +} SRowCompSupporter; + +static int compareRowData(const void *a, const void *b, const void *userData) { + const SResultRow *pRow1 = (const SResultRow *)a; + const SResultRow *pRow2 = (const SResultRow *)b; + + SRowCompSupporter *supporter = (SRowCompSupporter *)userData; + SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; + + SFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); + SFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); + + int16_t offset = supporter->dataOffset; + char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); + char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset); + + return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0; +} + +static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, SSDataBlock* pDataBlock) { + SArray *columnOrderList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); + size_t size = taosArrayGetSize(columnOrderList); + taosArrayDestroy(columnOrderList); + + if (size <= 0) { + return; + } + + int32_t orderId = pRuntimeEnv->pQueryAttr->order.orderColId; + if (orderId <= 0) { + return; + } + + bool found = false; + int16_t dataOffset = 0; + + for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j); + if (orderId == j) { + found = true; + break; + } + + dataOffset += pColInfoData->info.bytes; + } + + if (found == false) { + return; + } + + int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resSchema.type; + + SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)}; + taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support); +} + +//setup the output buffer for each operator +SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { + const static int32_t minSize = 8; + + SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = numOfOutput; + + res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = pExpr[i].base.resSchema.type; + idata.info.bytes = pExpr[i].base.resSchema.bytes; + idata.info.colId = pExpr[i].base.resSchema.colId; + + int32_t size = MAX(idata.info.bytes * numOfRows, minSize); + idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform + taosArrayPush(res->pDataBlock, &idata); + } + + return res; +} + +void* destroyOutputBuf(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return NULL; + } + + int32_t numOfOutput = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + tfree(pBlock); + return NULL; +} + +//int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// bool hasMainFunction = hasMainOutput(pQueryAttr); +// +// int32_t maxOutput = 0; +// for (int32_t j = 0; j < numOfOutput; ++j) { +// int32_t id = pCtx[j].functionId; +// +// /* +// * ts, tag, tagprj function can not decide the output number of current query +// * the number of output result is decided by main output +// */ +// if (hasMainFunction && (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ)) { +// continue; +// } +// +// SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); +// if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { +// maxOutput = pResInfo->numOfRes; +// } +// } +// +// assert(maxOutput >= 0); +// return maxOutput; +//} +// +//static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +// for (int32_t j = 0; j < numOfOutput; ++j) { +// SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); +// pResInfo->numOfRes = 0; +// } +//} + +static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { + return true; +// bool hasTags = false; +// int32_t numOfSelectivity = 0; +// +// for (int32_t i = 0; i < numOfOutput; ++i) { +// int32_t functId = pCtx[i].functionId; +// if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) { +// hasTags = true; +// continue; +// } +// +// if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) { +// numOfSelectivity++; +// } +// } +// +// return (numOfSelectivity > 0 && hasTags); +} + +static bool isProjQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functId != FUNCTION_PRJ && functId != FUNCTION_TAGPRJ) { + return false; + } + } + + return true; +} + +static bool hasNull(SColIndex* pColIndex, SColumnDataAgg *pStatis) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + return false; + } + + if (pStatis != NULL && pStatis->numOfNull == 0) { + return false; + } + + return true; +} + +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { + // more than the capacity, reallocate the resources + if (pResultRowInfo->size < pResultRowInfo->capacity) { + return; + } + + int64_t newCapacity = 0; + if (pResultRowInfo->capacity > 10000) { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); + } else { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); + } + + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pResultRowInfo->pResult = (SResultRow **)t; + + int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; + memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); + + pResultRowInfo->capacity = (int32_t)newCapacity; +} + +static bool chkResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, + int16_t bytes, bool masterscan, uint64_t uid) { + bool existed = false; + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return p1 != NULL; + } + + if (p1 != NULL) { + if (pResultRowInfo->size == 0) { + existed = false; + assert(pResultRowInfo->curPos == -1); + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); + int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (index != NULL) { + existed = true; + } else { + existed = false; + } + } + } + + return existed; + } + + return p1 != NULL; +} + + +static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, int64_t tid, + char* pData, int16_t bytes, bool masterscan, uint64_t tableGroupId) { + bool existed = false; + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tableGroupId); + + SResultRow **p1 = + (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + // in case of repeat scan/reverse scan, no new time window added. + if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQueryAttr)) { + if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. + return (p1 != NULL)? *p1:NULL; + } + + if (p1 != NULL) { + if (pResultRowInfo->size == 0) { + existed = false; + assert(pResultRowInfo->curPos == -1); + } else if (pResultRowInfo->size == 1) { + existed = (pResultRowInfo->pResult[0] == (*p1)); + pResultRowInfo->curPos = 0; + } else { // check if current pResultRowInfo contains the existed pResultRow + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); + int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + if (index != NULL) { + pResultRowInfo->curPos = (int32_t) *index; + existed = true; + } else { + existed = false; + } + } + } + } else { + // In case of group by column query, the required SResultRow object must be existed in the pResultRowInfo object. + if (p1 != NULL) { + return *p1; + } + } + + if (!existed) { + prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); + + SResultRow *pResult = NULL; + if (p1 == NULL) { + pResult = getNewResultRow(pRuntimeEnv->pool); + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // add a new result set for a new group + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; + taosArrayPush(pRuntimeEnv->pResultRowArrayList, &cell); + } else { + pResult = *p1; + } + + pResultRowInfo->curPos = pResultRowInfo->size; + pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + + int64_t index = pResultRowInfo->curPos; + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); + taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + } + + // too many time window in query + if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + + return pResultRowInfo->pResult[pResultRowInfo->curPos]; +} + +static void getInitialStartTimeWindow(SQueryAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQueryAttr, ts, pQueryAttr->window.ekey, ts, w); + + int64_t key = w->skey; + while(key < ts) { // moving towards end + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + key = taosTimeAdd(key, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision); + } else { + key += pQueryAttr->interval.sliding; + } + + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + +// get the correct time window according to the handled timestamp +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { + STimeWindow w = {0}; + + if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pQueryAttr, ts, &w); + + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + w.ekey = w.skey + pQueryAttr->interval.interval - 1; + } + } else { + w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + } + + if (w.skey > ts || w.ekey < ts) { + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, &pQueryAttr->interval, pQueryAttr->precision); + w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + } + + int64_t et = st + pQueryAttr->interval.interval - 1; + if (et < ts) { + st += ((ts - et + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + } + + w.skey = st; + w.ekey = w.skey + pQueryAttr->interval.interval - 1; + } + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + w.ekey = pQueryAttr->window.ekey; + } + + return w; +} + +// get the correct time window according to the handled timestamp +static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQueryAttr *pQueryAttr) { + STimeWindow w = {0}; + + if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value + getInitialStartTimeWindow(pQueryAttr, ts, &w); + + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + w.ekey = w.skey + pQueryAttr->interval.interval - 1; + } + } else { + w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; + } + + /* + * query border check, skey should not be bounded by the query time range, since the value skey will + * be used as the time window index value. So we only change ekey of time window accordingly. + */ + if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + w.ekey = pQueryAttr->window.ekey; + } + + return w; +} + +// a new buffer page for each table. Needs to opt this design +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { + if (pWindowRes->pageId != -1) { + return 0; + } + + SFilePage *pData = NULL; + + // in the first scan, new space needed for results + int32_t pageId = -1; + SIDList list = getDataBufPagesIdList(pResultBuf, tid); + + if (taosArrayGetSize(list) == 0) { + pData = getNewDataBuf(pResultBuf, tid, &pageId); + } else { + SPageInfo* pi = getLastPageInfo(list); + pData = getResBufPage(pResultBuf, pi->pageId); + pageId = pi->pageId; + + if (pData->num + size > pResultBuf->pageSize) { + // release current page first, and prepare the next one + releaseResBufPageInfo(pResultBuf, pi); + pData = getNewDataBuf(pResultBuf, tid, &pageId); + if (pData != NULL) { + assert(pData->num == 0); // number of elements must be 0 for new allocated buffer + } + } + } + + if (pData == NULL) { + return -1; + } + + // set the number of rows in current disk page + if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer + pWindowRes->pageId = pageId; + pWindowRes->offset = (int32_t)pData->num; + + pData->num += size; + assert(pWindowRes->pageId >= 0); + } + + return 0; +} + +static bool chkWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + assert(win->skey <= win->ekey); + return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); +} + +static int32_t setResultOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + assert(win->skey <= win->ekey); + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId); + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // not assign result buffer yet, add new result buffer + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) tableGroupId, pRuntimeEnv->pQueryAttr->intermediateResultRowSize); + if (ret != TSDB_CODE_SUCCESS) { + return -1; + } + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + + return TSDB_CODE_SUCCESS; +} + +static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + pResult->startInterp = true; + } else { + pResult->endInterp = true; + } +} + +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + return pResult->startInterp == true; + } else { + return pResult->endInterp == true; + } +} + +static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, + int16_t order, int64_t *pData) { + int32_t forwardStep = 0; + + if (order == TSDB_ORDER_ASC) { + int32_t end = searchFn((char*) &pData[pos], numOfRows - pos, ekey, order); + if (end >= 0) { + forwardStep = end; + + if (pData[end + pos] == ekey) { + forwardStep += 1; + } + } + } else { + int32_t end = searchFn((char *)pData, pos + 1, ekey, order); + if (end >= 0) { + forwardStep = pos - end; + + if (pData[end] == ekey) { + forwardStep += 1; + } + } + } + + assert(forwardStep >= 0); + return forwardStep; +} + +static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { + int64_t skey = TSKEY_INITIAL_VAL; + int32_t i = 0; + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } + + // new closed result rows + if (timeWindowInterpo) { + if (pResult->endInterp && ((pResult->win.skey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery))) { + if (i > 0) { // the first time window, the startInterp is false. + assert(pResult->startInterp); + } + + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } + } else { + if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) { + closeResultRow(pResultRowInfo, i); + } else { + skey = pResult->win.skey; + } + } + } + + // all result rows are closed, set the last one to be the skey + if (skey == TSKEY_INITIAL_VAL) { + if (pResultRowInfo->size == 0) { +// assert(pResultRowInfo->current == NULL); + assert(pResultRowInfo->curPos == -1); + pResultRowInfo->curPos = -1; + } else { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } + } else { + + for (i = pResultRowInfo->size - 1; i >= 0; --i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + if (pResult->closed) { + break; + } + } + + if (i == pResultRowInfo->size - 1) { + pResultRowInfo->curPos = i; + } else { + pResultRowInfo->curPos = i + 1; // current not closed result object + } + } + + //pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; +} + +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { + closeAllResultRows(pResultRowInfo); + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } else { + int32_t step = ascQuery ? 1 : -1; + doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); + } +} + +static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { + assert(startPos >= 0 && startPos < pDataBlockInfo->rows); + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + STableQueryInfo* item = pRuntimeEnv->current; + + int32_t num = -1; + int32_t order = pQueryAttr->order.order; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (updateLastKey) { // update the last key + item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; + } + } else { + num = pDataBlockInfo->rows - startPos; + if (updateLastKey) { + item->lastKey = pDataBlockInfo->window.ekey + step; + } + } + } else { // desc + if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { + num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); + if (updateLastKey) { // update the last key + item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; + } + } else { + num = startPos + 1; + if (updateLastKey) { + item->lastKey = pDataBlockInfo->window.skey + step; + } + } + } + + assert(num >= 0); + return num; +} + +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { +#if 0 + int32_t output = 0; + + if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { + //qError("empty udf function, type:%d", type); + return; + } + +// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); + + switch (type) { + case TSDB_UDF_FUNC_NORMAL: + if (pUdfInfo->isScript) { + (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, + (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, + (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes); + } else { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); + + (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, + pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init); + } + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pCtx->resultInfo->numOfRes = output; + } else { + pCtx->resultInfo->numOfRes += output; + } + + if (pCtx->resultInfo->numOfRes > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + + case TSDB_UDF_FUNC_MERGE: + if (pUdfInfo->isScript) { + (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); + } else { + (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); + } + + // set the output value exist + pCtx->resultInfo->numOfRes = output; + if (output > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + + case TSDB_UDF_FUNC_FINALIZE: { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); + if (pUdfInfo->isScript) { + (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); + } else { + (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); + } + // set the output value exist + pCtx->resultInfo->numOfRes = output; + if (output > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + } + } +#endif + +} + +static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, + int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + bool hasAggregates = pCtx[0].isAggSet; + + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].size = forwardStep; + pCtx[k].startTs = pWin->skey; + + // keep it temporarialy + char* start = pCtx[k].pInput; + + int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); + if (pCtx[k].pInput != NULL) { + pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; + } + + if (tsCol != NULL) { + pCtx[k].ptsList = &tsCol[pos]; + } + + // not a whole block involved in query processing, statistics data can not be used + // NOTE: the original value of isSet have been changed here + if (pCtx[k].isAggSet && forwardStep < numOfTotal) { + pCtx[k].isAggSet = false; + } + + int32_t functionId = pCtx[k].functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { +// if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo +// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; +// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); +// } else { +// aAggs[functionId].xFunction(&pCtx[k]); +// } + } + + // restore it + pCtx[k].isAggSet = hasAggregates; + pCtx[k].pInput = start; + } +} + + +static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, + TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { + getNextTimeWindow(pQueryAttr, pNext); + + // next time window is not in current block + if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || + (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + return -1; + } + + TSKEY startKey = -1; + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + startKey = pNext->skey; + if (startKey < pQueryAttr->window.skey) { + startKey = pQueryAttr->window.skey; + } + } else { + startKey = pNext->ekey; + if (startKey > pQueryAttr->window.skey) { + startKey = pQueryAttr->window.skey; + } + } + + int32_t startPos = 0; + + // tumbling time window query, a special case of sliding time window query + if (pQueryAttr->interval.sliding == pQueryAttr->interval.interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + startPos = prevPosition + factor; + } else { + if (startKey <= pDataBlockInfo->window.skey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + startPos = 0; + } else if (startKey >= pDataBlockInfo->window.ekey && !QUERY_IS_ASC_QUERY(pQueryAttr)) { + startPos = pDataBlockInfo->rows - 1; + } else { + startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQueryAttr->order.order); + } + } + + /* interp query with fill should not skip time window */ + if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { + return startPos; + } + + /* + * This time window does not cover any data, try next time window, + * this case may happen when the time window is too small + */ + if (primaryKeys == NULL) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + assert(pDataBlockInfo->window.skey <= pNext->ekey); + } else { + assert(pDataBlockInfo->window.ekey >= pNext->skey); + } + } else { + if (QUERY_IS_ASC_QUERY(pQueryAttr) && primaryKeys[startPos] > pNext->ekey) { + TSKEY next = primaryKeys[startPos]; + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); + pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + pNext->ekey += ((next - pNext->ekey + pQueryAttr->interval.sliding - 1)/pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + pNext->skey = pNext->ekey - pQueryAttr->interval.interval + 1; + } + } else if ((!QUERY_IS_ASC_QUERY(pQueryAttr)) && primaryKeys[startPos] < pNext->skey) { + TSKEY next = primaryKeys[startPos]; + if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); + pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + pNext->skey -= ((pNext->skey - next + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + pNext->ekey = pNext->skey + pQueryAttr->interval.interval - 1; + } + } + } + + return startPos; +} + +static FORCE_INLINE TSKEY reviseWindowEkey(SQueryAttr *pQueryAttr, STimeWindow *pWindow) { + TSKEY ekey = -1; + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + ekey = pWindow->ekey; + if (ekey > pQueryAttr->window.ekey) { + ekey = pQueryAttr->window.ekey; + } + } else { + ekey = pWindow->skey; + if (ekey < pQueryAttr->window.ekey) { + ekey = pQueryAttr->window.ekey; + } + } + + return ekey; +} + +static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; + } + } else { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; + } + } +} + +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, + int32_t rowIndex) { + if (pDataBlock == NULL) { + return; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + } +} + +static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { + TSKEY ts = TSKEY_INITIAL_VAL; + + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + if (tsCols == NULL) { + ts = ascQuery? win->skey : win->ekey; + } else { + int32_t offset = ascQuery? 0:rows-1; + ts = tsCols[offset]; + } + + return ts; +} + +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + } +} + +void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + if (pCtx[0].functionId == FUNCTION_ARITHM) { +// SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; +// if (pSupport->colList == NULL) { +// doSetInputDataBlock(pOperator, pCtx, pBlock, order); +// } else { +// doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); +// } + } else { + if (pBlock->pDataBlock != NULL) { + doSetInputDataBlock(pOperator, pCtx, pBlock, order); + } else { + doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); + } + } +} + +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; + + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + + if (pCtx[i].functionId == FUNCTION_ARITHM) { +// setArithParams((SScalarFunctionSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); + } else { + SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; + if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == FUNCTION_BLKINFO) || + (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { + SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + + // in case of the block distribution query, the inputBytes is not a constant value. + pCtx[i].pInput = p->pData; + assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); + + if (pCtx[i].functionId < 0) { + SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); + pCtx[i].ptsList = (int64_t*) tsInfo->pData; + + continue; + } + +// uint32_t status = aAggs[pCtx[i].functionId].status; +// if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { +// SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); +// // In case of the top/bottom query again the nest query result, which has no timestamp column +// // don't set the ptsList attribute. +// if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { +// pCtx[i].ptsList = (int64_t*) tsInfo->pData; +// } else { +// pCtx[i].ptsList = NULL; +// } +// } + } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { + SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + + pCtx[i].pInput = p->pData; + assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); + for(int32_t j = 0; j < pBlock->info.rows; ++j) { + char* dst = p->pData + j * p->info.bytes; + taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); + } + } + } + } +} + +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { + pCtx[k].startTs = startTs;// this can be set during create the struct + + int32_t functionId = pCtx[k].functionId; +// if (functionId < 0) { +// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; +// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); +// } else { +// aAggs[functionId].xFunction(&pCtx[k]); +// } + } + } +} + +static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].startTs = pQueryAttr->window.skey; + + // Always set the asc order for merge stage process + if (pCtx[k].currentStage == MERGE_STAGE) { + pCtx[k].order = TSDB_ORDER_ASC; + } + + pCtx[k].startTs = pQueryAttr->window.skey; + + if (pCtx[k].functionId < 0) { + // load the script and exec +// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; +// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); +// } else { +// aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); + } + } +} + +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SExprInfo* pExpr = pOperator->pExpr; + + SQLFunctionCtx* pCtx = pInfo->pCtx; + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex * pColIndex = &pExpr[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); + + assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + if (functionId == FUNCTION_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; + + if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { + if (prevRowIndex == -1) { + pCtx[k].start.ptr = (char *)pRuntimeEnv->prevRow[index]; + } else { + pCtx[k].start.ptr = (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; + } + + pCtx[k].end.ptr = (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes; + } + } + } else if (functionId == FUNCTION_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v }; + + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; + } + } + } +} + +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; + + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = ascQuery? win->skey:win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step]; + + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, + key, RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, + int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + TSKEY actualEndKey = tsCols[endRowIndex]; + + TSKEY key = QUERY_IS_ASC_QUERY(pQueryAttr)? win->ekey:win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQueryAttr)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return false; + } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; +} + +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + if (!pQueryAttr->timeWindowInterpo) { + return; + } + + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + + if (pBlock->pDataBlock == NULL){ +// tscError("pBlock->pDataBlock == NULL"); + return; + } + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { // it is not interpolated, now start to generated the interpolated value + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock, + tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + if (pQueryAttr->pointInterpQuery) { + return; + } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQueryAttr)? pBlock->info.window.ekey:pBlock->info.window.skey; + bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } else { + setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_END_INTERP); + } +} + +static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + int32_t prevIndex = pResultRowInfo->curPos; + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + } + + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t forwardStep = 0; + TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + forwardStep = + getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // prev time window not interpolation yet. + int32_t curIndex = pResultRowInfo->curPos; + if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = getResultRow(pResultRowInfo, j); + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } + + STimeWindow w = pRes->win; + ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, + tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, + tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + // restore current time window + ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQueryAttr, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ekey = reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + if (pQueryAttr->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +} + + +static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + } + + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t forwardStep = 0; + int32_t ret = 0; + STimeWindow preWin = win; + + while (1) { + // null data, failed to allocate more memory buffer + ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, + tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + preWin = win; + + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { + int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + startPos = pSDataBlock->info.rows - 1; + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + break; + } + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + + if (pQueryAttr->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +} + + + +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->current; + + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + + if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + return; + } + + SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); + int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; + + STimeWindow w = TSWINDOW_INITIALIZER; + + int32_t num = 0; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + char* val = ((char*)pColInfoData->pData) + bytes * j; + if (isNull(val, type)) { + continue; + } + + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); + memcpy(pInfo->prevData, val, bytes); + num++; + continue; + } + + if (IS_VAR_DATA_TYPE(type)) { + int32_t len = varDataLen(val); + if(len == varDataLen(pInfo->prevData) && memcmp(varDataVal(pInfo->prevData), varDataVal(val), len) == 0) { + num++; + continue; + } + } else { + if (memcmp(pInfo->prevData, val, bytes) == 0) { + num++; + continue; + } + } + + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); + } + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + + num = 1; + memcpy(pInfo->prevData, val, bytes); + } + + if (num > 0) { + char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); + memcpy(pInfo->prevData, val, bytes); + + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); + } + + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + } + + tfree(pInfo->prevData); +} + +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->current; + + // primary timestamp column + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + int64_t gap = pOperator->pRuntimeEnv->pQueryAttr->sw.gap; + pInfo->numOfRows = 0; + if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { + pInfo->reptScan = true; + pInfo->prevTs = INT64_MIN; + } + + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + if (pInfo->prevTs == INT64_MIN) { + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } else if (tsList[j] - pInfo->prevTs <= gap && (tsList[j] - pInfo->prevTs) >= 0) { + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows += 1; + if (j == 0 && pInfo->start != 0) { + pInfo->numOfRows = 1; + pInfo->start = 0; + } + } else { // start a new session window + SResultRow* pResult = NULL; + + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); + + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } + } + + SResultRow* pResult = NULL; + + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); +} + +static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { + if (IS_VAR_DATA_TYPE(type)) { + if (pResultRow->key == NULL) { + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); + } else { + assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); + } + } else { + int64_t v = -1; + GET_TYPED_DATA(v, int64_t, type, pData); + + pResultRow->win.skey = v; + pResultRow->win.ekey = v; + } +} + +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; + SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; + SQLFunctionCtx *pCtx = binfo->pCtx; + + // not assign result buffer yet, add new result buffer, TODO remove it + char* d = pData; + int16_t len = bytes; + if (IS_VAR_DATA_TYPE(type)) { + d = varDataVal(pData); + len = varDataLen(pData); + } + + int64_t tid = 0; + SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); + assert (pResultRow != NULL); + + setResultRowKey(pResultRow, pData, type); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); + if (ret != 0) { + return -1; + } + } + + setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); + initCtxOutputBuffer(pCtx, numOfCols); + return TSDB_CODE_SUCCESS; +} + +static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { + size_t num = taosArrayGetSize(pGroupbyExpr->columnInfo); + for (int32_t k = 0; k < num; ++k) { + SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); + if (TSDB_COL_IS_TAG(pColIndex->flag)) { + continue; + } + + int32_t colId = pColIndex->colId; + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + if (pColInfo->info.colId == colId) { + return i; + } + } + } + + assert(0); + return -1; +} + +static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { + struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + // in case of timestamp column, always generated results. + int32_t functionId = pCtx->functionId; + if (functionId == FUNCTION_TS) { + return true; + } + + if (isRowEntryCompleted(pResInfo) || functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + return false; + } + + if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_FIRST) { + return QUERY_IS_ASC_QUERY(pQueryAttr); + } + + // denote the order type + if ((functionId == FUNCTION_LAST_DST || functionId == FUNCTION_LAST)) { + return pCtx->param[0].i == pQueryAttr->order.order; + } + + // in the reverse table scan, only the following functions need to be executed + if (IS_REVERSE_SCAN(pRuntimeEnv) || + (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != FUNCTION_STDDEV && functionId != FUNCTION_PERCT)) { + return false; + } + + return true; +} + +void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { + SColumnDataAgg *pAgg = NULL; + + if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + pAgg = &pSDataBlock->pBlockAgg[pColIndex->colIndex]; + + pCtx->agg = *pAgg; + pCtx->isAggSet = true; + assert(pCtx->agg.numOfNull <= pSDataBlock->info.rows); + } else { + pCtx->isAggSet = false; + } + + pCtx->hasNull = hasNull(pColIndex, pAgg); + + // set the statistics data for primary time stamp column + if (pCtx->functionId == FUNCTION_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pCtx->isAggSet = true; + pCtx->agg.min = pSDataBlock->info.window.skey; + pCtx->agg.max = pSDataBlock->info.window.ekey; + } +} + +// set the output buffer for the selectivity + tag query +static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { + if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { + return TSDB_CODE_SUCCESS; + } + + int32_t num = 0; + int16_t tagLen = 0; + + SQLFunctionCtx* p = NULL; + SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + + if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + tagLen += pCtx[i].outputBytes; + pTagCtx[num++] = &pCtx[i]; + } else if (1/*(aAggs[functionId].status & FUNCSTATE_SELECTIVITY) != 0*/) { + p = &pCtx[i]; + } else if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column + continue; + } else { + // the column may be the normal column, group by normal_column, the functionId is FUNCTION_PRJ + } + } + if (p != NULL) { + p->tagInfo.pTagCtxList = pTagCtx; + p->tagInfo.numOfTagCols = num; + p->tagInfo.tagsLen = tagLen; + } else { + tfree(pTagCtx); + } + + return TSDB_CODE_SUCCESS; +} + +static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, + int32_t** rowCellInfoOffset) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + if (pFuncCtx == NULL) { + return NULL; + } + + *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + if (*rowCellInfoOffset == 0) { + tfree(pFuncCtx); + return NULL; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + SSqlExpr *pSqlExpr = &pExpr[i].base; + SQLFunctionCtx* pCtx = &pFuncCtx[i]; + + SColIndex *pIndex = &pSqlExpr->colInfo; + + if (TSDB_COL_REQ_NULL(pIndex->flag)) { + pCtx->requireNull = true; + pIndex->flag &= ~(TSDB_COL_NULL); + } else { + pCtx->requireNull = false; + } + +// pCtx->inputBytes = pSqlExpr->colBytes; +// pCtx->inputType = pSqlExpr->colType; + + pCtx->ptsOutputBuf = NULL; + + pCtx->outputBytes = pSqlExpr->resSchema.bytes; + pCtx->outputType = pSqlExpr->resSchema.type; + + pCtx->order = pQueryAttr->order.order; +// pCtx->functionId = pSqlExpr->functionId; + pCtx->stableQuery = pQueryAttr->stableQuery; + pCtx->interBufBytes = pSqlExpr->interBytes; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; + + pCtx->numOfParams = pSqlExpr->numOfParams; + for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + int16_t type = pSqlExpr->param[j].nType; + int16_t bytes = pSqlExpr->param[j].nLen; +// if (pSqlExpr->functionId == FUNCTION_STDDEV_DST) { +// continue; +// } + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + taosVariantCreateFromBinary(&pCtx->param[j], pSqlExpr->param[j].pz, bytes, type); + } else { + taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlExpr->param[j].i, bytes, type); + } + } + + // set the order information for top/bottom query + int32_t functionId = pCtx->functionId; + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + int32_t f = getExprFunctionId(&pExpr[0]); + assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); + + pCtx->param[2].i = pQueryAttr->order.order; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[3].i = functionId; + pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + + pCtx->param[1].i = pQueryAttr->order.orderColId; + } else if (functionId == FUNCTION_INTERP) { + pCtx->param[2].i = (int8_t)pQueryAttr->fillType; + if (pQueryAttr->fillVal != NULL) { + if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value + if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); + } + } + } + } else if (functionId == FUNCTION_TS_COMP) { + pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == FUNCTION_TWA) { + pCtx->param[1].i = pQueryAttr->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].i = pQueryAttr->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == FUNCTION_ARITHM) { + pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); + } + } + +// for(int32_t i = 1; i < numOfOutput; ++i) { +// (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr[i - 1].base.interBytes); +// } + + setCtxTagColumnInfo(pFuncCtx, numOfOutput); + + return pFuncCtx; +} + +static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + if (pCtx == NULL) { + return NULL; + } + + for (int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { + taosVariantDestroy(&pCtx[i].param[j]); + } + + taosVariantDestroy(&pCtx[i].tag); + tfree(pCtx[i].tagInfo.pTagCtxList); + } + + tfree(pCtx); + return NULL; +} + +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { + //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + pRuntimeEnv->prevGroupId = INT32_MIN; + pRuntimeEnv->pQueryAttr = pQueryAttr; + + pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables * 10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); + pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); + pRuntimeEnv->pResultRowArrayList = taosArrayInit(numOfTables, sizeof(SResultRowCell)); + + pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); + pRuntimeEnv->tagVal = malloc(pQueryAttr->tagLen); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + + pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput); + + if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || + pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { + goto _clean; + } + + if (pQueryAttr->numOfCols) { + char* start = POINTER_BYTES * pQueryAttr->numOfCols + (char*) pRuntimeEnv->prevRow; + pRuntimeEnv->prevRow[0] = start; + for(int32_t i = 1; i < pQueryAttr->numOfCols; ++i) { + pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQueryAttr->tableCols[i-1].bytes; + } + + if (pQueryAttr->tableCols[0].type == TSDB_DATA_TYPE_TIMESTAMP) { + *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; + } + } + + //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); + + // group by normal column, sliding window query, interval query are handled by interval query processor + // interval (down sampling operation) + int32_t numOfOperator = (int32_t) taosArrayGetSize(pOperator); + for(int32_t i = 0; i < numOfOperator; ++i) { + int32_t* op = taosArrayGet(pOperator, i); + + switch (*op) { + case OP_TagScan: { + pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + break; + } + case OP_MultiTableTimeInterval: { + pRuntimeEnv->proot = + createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + break; + } + case OP_AllMultiTableTimeInterval: { + pRuntimeEnv->proot = + createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + break; + } + case OP_TimeWindow: { + pRuntimeEnv->proot = + createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_AllTimeWindow: { + pRuntimeEnv->proot = + createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_Groupby: { + pRuntimeEnv->proot = + createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_SessionWindow: { + pRuntimeEnv->proot = + createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + case OP_MultiTableAggregate: { + pRuntimeEnv->proot = + createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + break; + } + case OP_Aggregate: { + pRuntimeEnv->proot = + createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput && opType != OP_Join) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + + case OP_Project: { // TODO refactor to remove arith operator. + SOperatorInfo* prev = pRuntimeEnv->proot; + if (i == 0) { + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor + setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); + } + } else { + prev = pRuntimeEnv->proot; + assert(pQueryAttr->pExpr2 != NULL); + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); + } + break; + } + + case OP_StateWindow: { + pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; + if (opType != OP_DummyInput) { + setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); + } + break; + } + + case OP_Limit: { + pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); + break; + } + + case OP_Filter: { // todo refactor + int32_t numOfFilterCols = 0; + if (pQueryAttr->stableQuery) { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, + pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); + } else { + SColumnInfo* pColInfo = + extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); + pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, + pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); + freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); + } + + break; + } + + case OP_Fill: { + SOperatorInfo* pInfo = pRuntimeEnv->proot; + pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); + break; + } + + case OP_MultiwayMergeSort: { + pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); + break; + } + + case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. + bool multigroupResult = pQueryAttr->multigroupResult; + if (pQueryAttr->multigroupResult) { + multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); + } + + pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, + pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); + break; + } + + case OP_SLimit: { + int32_t num = pRuntimeEnv->proot->numOfOutput; + SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; + pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); + break; + } + + case OP_Distinct: { + pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + break; + } + + case OP_Order: { + pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); + break; + } + + default: { + assert(0); + } + } + } + + return TSDB_CODE_SUCCESS; + +_clean: + destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput); + tfree(pRuntimeEnv->pResultRowHashTable); + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); + + return TSDB_CODE_QRY_OUT_OF_MEMORY; +} + +static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + +// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + pRuntimeEnv->pQueryHandle = NULL; + +// SMemRef* pMemRef = &pQueryAttr->memRef; +// assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); +} + +static void destroyTsComp(SQueryRuntimeEnv *pRuntimeEnv, SQueryAttr *pQueryAttr) { + if (pQueryAttr->tsCompQuery && pRuntimeEnv->outputBuf && pRuntimeEnv->outputBuf->pDataBlock && taosArrayGetSize(pRuntimeEnv->outputBuf->pDataBlock) > 0) { + SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); + if (pColInfoData) { + FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor + if (f) { + fclose(f); + *(FILE **)pColInfoData->pData = NULL; + } + } + } +} + +static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; + + //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); + + destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); + destroyUdfInfo(pRuntimeEnv->pUdfInfo); + destroyResultBuf(pRuntimeEnv->pResultBuf); + doFreeQueryHandle(pRuntimeEnv); + + destroyTsComp(pRuntimeEnv, pQueryAttr); + + pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); + + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); + + taosHashCleanup(pRuntimeEnv->pResultRowHashTable); + pRuntimeEnv->pResultRowHashTable = NULL; + + taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); + pRuntimeEnv->pTableRetrieveTsMap = NULL; + + taosHashCleanup(pRuntimeEnv->pResultRowListSet); + pRuntimeEnv->pResultRowListSet = NULL; + + destroyOperatorInfo(pRuntimeEnv->proot); + + pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); + taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); + taosArrayDestroy(pRuntimeEnv->pResultRowArrayList); + pRuntimeEnv->prevResult = NULL; +} + +static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { + return pQInfo->rspContext != NULL; +} + +bool isQueryKilled(SQInfo *pQInfo) { + if (IS_QUERY_KILLED(pQInfo)) { + return true; + } + + // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived + // abort current query execution. + if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && + (!needBuildResAfterQueryComplete(pQInfo))) { + + assert(pQInfo->startExecTs != 0); + //qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 +// ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); + return true; + } + + return false; +} + +void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} + +//static bool isFixedOutputQuery(SQueryAttr* pQueryAttr) { +// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { +// return false; +// } +// +// // Note:top/bottom query is fixed output query +// if (pQueryAttr->topBotQuery || pQueryAttr->groupbyColumn || pQueryAttr->tsCompQuery) { +// return true; +// } +// +// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { +// SSqlExpr *pExpr = &pQueryAttr->pExpr1[i].base; +// +// if (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_TS_DUMMY) { +// continue; +// } +// +// if (!IS_MULTIOUTPUT(aAggs[pExpr->functionId].status)) { +// return true; +// } +// } +// +// return false; +//} + +// todo refactor with isLastRowQuery +//bool isPointInterpoQuery(SQueryAttr *pQueryAttr) { +// for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { +// int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; +// if (functionId == FUNCTION_INTERP) { +// return true; +// } +// } +// +// return false; +//} + +static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionID = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functionID == FUNCTION_LAST_ROW) { + return true; + } + } + + return false; +} + +static bool isCachedLastQuery(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { + continue; + } + + return false; + } + + if (pQueryAttr->order.order != TSDB_ORDER_DESC || !TSWINDOW_IS_EQUAL(pQueryAttr->window, TSWINDOW_DESC_INITIALIZER)) { + return false; + } + + if (pQueryAttr->groupbyColumn) { + return false; + } + + if (pQueryAttr->interval.interval > 0) { + return false; + } + + if (pQueryAttr->numOfFilterCols > 0 || pQueryAttr->havingNum > 0) { + return false; + } + + return true; +} + + + +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ +bool onlyQueryTags(SQueryAttr* pQueryAttr) { + for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; + + int32_t functionId = getExprFunctionId(pExprInfo); + + if (functionId != FUNCTION_TAGPRJ && + functionId != FUNCTION_TID_TAG && + (!(functionId == FUNCTION_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { + return false; + } + } + + return true; +} + +///////////////////////////////////////////////////////////////////////////////////////////// + +void getAlignQueryTimeWindow(SQueryAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { + assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); + win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); + + /* + * if the realSkey > INT64_MAX - pQueryAttr->interval.interval, the query duration between + * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. + */ + if (keyFirst > (INT64_MAX - pQueryAttr->interval.interval)) { + assert(keyLast - keyFirst < pQueryAttr->interval.interval); + win->ekey = INT64_MAX; + } else if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { + win->ekey = taosTimeAdd(win->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else { + win->ekey = win->skey + pQueryAttr->interval.interval - 1; + } +} + +/* + * todo add more parameters to check soon.. + */ +bool colIdCheck(SQueryAttr *pQueryAttr, uint64_t qId) { + // load data column information is incorrect + for (int32_t i = 0; i < pQueryAttr->numOfCols - 1; ++i) { + if (pQueryAttr->tableCols[i].colId == pQueryAttr->tableCols[i + 1].colId) { + //qError("QInfo:0x%"PRIx64" invalid data load column for query", qId); + return false; + } + } + + return true; +} + +// todo ignore the avg/sum/min/max/count/stddev/top/bottom functions, of which +// the scan order is not matter +static bool onlyOneQueryType(SQueryAttr *pQueryAttr, int32_t functId, int32_t functIdDst) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + + if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || + functionId == FUNCTION_TAG_DUMMY) { + continue; + } + + if (functionId != functId && functionId != functIdDst) { + return false; + } + } + + return true; +} + +static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_FIRST, FUNCTION_FIRST_DST); } + +static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, FUNCTION_LAST, FUNCTION_LAST_DST); } + +static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); } + +static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { + bool hasFirstLastFunc = false; + bool hasOtherFunc = false; + + if (status == BLK_DATA_ALL_NEEDED || status == BLK_DATA_DISCARD) { + return status; + } + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQuery->pExpr1[i]); + + if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || + functionId == FUNCTION_TAG_DUMMY) { + continue; + } + + if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { + hasFirstLastFunc = true; + } else { + hasOtherFunc = true; + } + } + + if (hasFirstLastFunc && status == BLK_DATA_NO_NEEDED) { + if(!hasOtherFunc) { + return BLK_DATA_DISCARD; + } else { + return BLK_DATA_ALL_NEEDED; + } + } + + return status; +} + +static void doUpdateLastKey(SQueryAttr* pQueryAttr) { + STimeWindow* win = &pQueryAttr->window; + + size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); + for(int32_t i = 0; i < num; ++i) { + SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t len = taosArrayGetSize(p1); + for(int32_t j = 0; j < len; ++j) { +// STableKeyInfo* pInfo = taosArrayGet(p1, j); +// +// // update the new lastkey if it is equalled to the value of the old skey +// if (pInfo->lastKey == win->ekey) { +// pInfo->lastKey = win->skey; +// } + } + } +} + +static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { + SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + + // in case of point-interpolation query, use asc order scan + char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 + "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; + + // todo handle the case the the order irrelevant query type mixed up with order critical query type + // descending order query for last_row query + if (isFirstLastRowQuery(pQueryAttr)) { + //qDebug("QInfo:0x%"PRIx64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, pQueryAttr->order.order, TSDB_ORDER_ASC); + + pQueryAttr->order.order = TSDB_ORDER_ASC; + if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + } + + pQueryAttr->needReverseScan = false; + return; + } + + if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { + pQueryAttr->order.order = TSDB_ORDER_ASC; + if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + } + + pQueryAttr->needReverseScan = false; + doUpdateLastKey(pQueryAttr); + return; + } + + if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { + if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + } + + pQueryAttr->order.order = TSDB_ORDER_ASC; + return; + } + + if (pQueryAttr->interval.interval == 0) { + if (onlyFirstQuery(pQueryAttr)) { + if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, +// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_ASC; + pQueryAttr->needReverseScan = false; + } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, +// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_DESC; + pQueryAttr->needReverseScan = false; + } + + } else { // interval query + if (stableQuery) { + if (onlyFirstQuery(pQueryAttr)) { + if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, +// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_ASC; + pQueryAttr->needReverseScan = false; + } else if (onlyLastQuery(pQueryAttr)) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, +// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); + + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + doUpdateLastKey(pQueryAttr); + } + + pQueryAttr->order.order = TSDB_ORDER_DESC; + pQueryAttr->needReverseScan = false; + } + } + } +} + +static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t MIN_ROWS_PER_PAGE = 4; + + *rowsize = (int32_t)(pQueryAttr->resultRowSize * getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + int32_t overhead = sizeof(SFilePage); + + // one page contains at least two rows + *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; + while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { + *ps = ((*ps) << 1u); + } +} + +#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) + +//static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +// SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +// +// if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { +// return true; +// } +// +// return filterRangeExecute(pQueryAttr->pFilters, pDataStatis, pQueryAttr->numOfCols, numOfRows); +//} + +static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { + STimeWindow w = {0}; + + TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); + assert(w.ekey >= pBlockInfo->window.skey); + + if (w.ekey < pBlockInfo->window.ekey) { + return true; + } + + while(1) { + getNextTimeWindow(pQueryAttr, &w); + if (w.skey > pBlockInfo->window.ekey) { + break; + } + + assert(w.ekey > pBlockInfo->window.ekey); + if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { + return true; + } + } + } else { + getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); + assert(w.skey <= pBlockInfo->window.ekey); + + if (w.skey > pBlockInfo->window.skey) { + return true; + } + + while(1) { + getNextTimeWindow(pQueryAttr, &w); + if (w.ekey < pBlockInfo->window.skey) { + break; + } + + assert(w.skey < pBlockInfo->window.skey); + if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { + return true; + } + } + } + + return false; +} + +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); + +#if defined(_DEBUG_VIEW) + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag.i, pQueryAttr->order.order, pRuntimeEnv->pTsBuf->tsOrder, + pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); +#endif + + if (ascQuery) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } + + return TS_JOIN_TS_EQUAL; +} + +bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p) { + bool all = true; + + for (int32_t i = 0; i < numOfRows; ++i) { + bool qualified = false; + + for (int32_t k = 0; k < numOfFilterCols; ++k) { + char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; + + qualified = false; + for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { + SColumnFilterElem* pFilterElem = NULL; +// SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; + + bool isnull = isNull(pElem, pFilterInfo[k].info.type); + if (isnull) { +// if (pFilterElem->fp == isNullOperator) { +// qualified = true; +// break; +// } else { +// continue; +// } + } else { +// if (pFilterElem->fp == notNullOperator) { +// qualified = true; +// break; +// } else if (pFilterElem->fp == isNullOperator) { +// continue; +// } + } + + if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { + qualified = true; + break; + } + } + + if (!qualified) { + break; + } + } + + p[i] = qualified ? 1 : 0; + if (!qualified) { + all = false; + } + } + + return all; +} + +void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p) { + int32_t len = 0; + int32_t start = 0; + for (int32_t j = 0; j < numOfRows; ++j) { + if (p[j] == 1) { + len++; + } else { + if (len > 0) { + int32_t cstart = j - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, + len * bytes); + } + + start += len; + len = 0; + } + } + } + + if (len > 0) { + int32_t cstart = numOfRows - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; + } + + pBlock->info.rows = start; + pBlock->pBlockAgg = NULL; // clean the block statistics info + + if (start > 0) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && + pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; + pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); + } + } +} + +void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, + SSDataBlock* pBlock, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = calloc(numOfRows, sizeof(int8_t)); + bool all = true; + + if (pRuntimeEnv->pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; + } + + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } + } + + // save the cursor status + pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + } else { + all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p); + } + + if (!all) { + doCompactSDataBlock(pBlock, numOfRows, p); + } + + tfree(p); +} + +void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = NULL; + bool all = true; + + if (pRuntimeEnv->pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + p = calloc(numOfRows, sizeof(int8_t)); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; + } + + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } + } + + // save the cursor status + pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + } else { +// all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockAgg, pRuntimeEnv->pQueryAttr->numOfCols); + } + + if (!all) { + if (p) { + doCompactSDataBlock(pBlock, numOfRows, p); + } else { + pBlock->info.rows = 0; + pBlock->pBlockAgg = NULL; // clean the block statistics info + } + } + + tfree(p); +} + + + +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); +static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); + +static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { + SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + uint32_t status = BLK_DATA_NO_NEEDED; + + int32_t numOfOutput = pTableScanInfo->numOfOutput; + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; + + // group by + first/last should not apply the first/last block filter + if (functionId < 0) { + status |= BLK_DATA_ALL_NEEDED; + return status; + } else { +// status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); +// if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { +// return status; +// } + } + } + + return status; +} + +void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock) { + // set the initial static data value filter expression + for (int32_t i = 0; i < numOfFilterCols; ++i) { + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); + + if (pFilterInfo[i].info.colId == pColInfo->info.colId) { + pFilterInfo[i].pData = pColInfo->pData; + break; + } + } + } +} + +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, + uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + pBlock->pDataBlock = NULL; + pBlock->pBlockAgg = NULL; + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int64_t groupId = pRuntimeEnv->current->groupIndex; + bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + SQueryCostInfo* pCost = &pQInfo->summary; + + pCost->totalBlocks += 1; + pCost->totalRows += pBlock->info.rows; + + if (pRuntimeEnv->pTsBuf != NULL) { + (*status) = BLK_DATA_ALL_NEEDED; + + if (pQueryAttr->stableQuery) { // todo refactor + SExprInfo* pExprInfo = &pTableScanInfo->pExpr[0]; + int16_t tagId = (int16_t)pExprInfo->base.param[0].i; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagId); + + // compare tag first + SVariant t = {0}; + doSetTagValueInParam(pRuntimeEnv->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes); + setTimestampListJoinInfo(pRuntimeEnv, &t, pRuntimeEnv->current); + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); + if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (taosVariantCompare(&t, elem.tag) != 0))) { + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + + // Calculate all time windows that are overlapping or contain current data block. + // If current data block is contained by all possible time window, do not load current data block. + if (/*pQueryAttr->pFilters || */pQueryAttr->groupbyColumn || pQueryAttr->sw.gap > 0 || + (QUERY_IS_INTERVAL_QUERY(pQueryAttr) && overlapWithTimeWindow(pQueryAttr, &pBlock->info))) { + (*status) = BLK_DATA_ALL_NEEDED; + } + + // check if this data block is required to load + if ((*status) != BLK_DATA_ALL_NEEDED) { + bool needFilter = true; + + // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, + // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); + if (pQueryAttr->pointInterpQuery) { + needFilter = chkWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset); + } else { + if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate + doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, + pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, + pRuntimeEnv->current->groupIndex); + } + + if (needFilter) { + (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); + } else { + (*status) = BLK_DATA_ALL_NEEDED; + } + } + + SDataBlockInfo* pBlockInfo = &pBlock->info; + *status = updateBlockLoadStatus(pRuntimeEnv->pQueryAttr, *status); + + if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) { + //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, +// pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->discardBlocks += 1; + } else if ((*status) == BLK_DATA_STATIS_NEEDED) { + // this function never returns error? + pCost->loadBlockStatis += 1; +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); + + if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pCost->totalCheckedRows += pBlock->info.rows; + } + } else { + assert((*status) == BLK_DATA_ALL_NEEDED); + + // load the data block statistics to perform further filter + pCost->loadBlockStatis += 1; +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); + + if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { + { // set previous window + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr); + if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } + bool load = false; + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = pTableScanInfo->pCtx[i].functionId; + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { +// load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockAgg[i].min), +// (char*)&(pBlock->pBlockAgg[i].max)); + if (!load) { // current block has been discard due to filter applied + pCost->discardBlocks += 1; + //qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, +// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + } + + // current block has been discard due to filter applied +// if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockAgg, pTableScanInfo->pCtx, pBlockInfo->rows)) { +// pCost->discardBlocks += 1; +// qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, +// pBlockInfo->window.ekey, pBlockInfo->rows); +// (*status) = BLK_DATA_DISCARD; +// return TSDB_CODE_SUCCESS; +// } + + pCost->totalCheckedRows += pBlockInfo->rows; + pCost->loadBlocks += 1; +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// if (pBlock->pDataBlock == NULL) { +// return terrno; +// } + +// if (pQueryAttr->pFilters != NULL) { +// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); +// } + +// if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { +// filterColRowsInDataBlock(pRuntimeEnv, pBlock, ascQuery); +// } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + if (num <= 0) { + return -1; + } + + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY * keyList = (TSKEY *)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[lastPos]) return lastPos; + if (key == keyList[firstPos]) return firstPos; + if (key < keyList[firstPos]) return firstPos - 1; + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +/* + * set tag value in SQLFunctionCtx + * e.g.,tag information into input buffer + */ +static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes) { + taosVariantDestroy(tag); + + char* val = NULL; +// if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { +// val = tsdbGetTableName(pTable); +// assert(val != NULL); +// } else { +// val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); +// } + + if (val == NULL || isNull(val, type)) { + tag->nType = TSDB_DATA_TYPE_NULL; + return; + } + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t maxLen = bytes - VARSTR_HEADER_SIZE; + int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); + taosVariantCreateFromBinary(tag, varDataVal(val), len, type); + //taosVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); + } else { + taosVariantCreateFromBinary(tag, val, bytes, type); + } +} + +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { + assert(pTagColList != NULL && numOfTags > 0); + + for(int32_t i = 0; i < numOfTags; ++i) { + if (pTagColList[i].colId == colId) { + return &pTagColList[i]; + } + } + + return NULL; +} + +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + + SExprInfo *pExpr = pOperatorInfo->pExpr; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SExprInfo* pExprInfo = &pExpr[0]; + int32_t functionId = getExprFunctionId(pExprInfo); + + if (pQueryAttr->numOfOutput == 1 && functionId == FUNCTION_TS_COMP && pQueryAttr->stableQuery) { + assert(pExprInfo->base.numOfParams == 1); + + int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); + + doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); + return; + } else { + // set tag value, by which the results are aggregated. + int32_t offset = 0; + memset(pRuntimeEnv->tagVal, 0, pQueryAttr->tagLen); + + for (int32_t idx = 0; idx < numOfOutput; ++idx) { + SExprInfo* pLocalExprInfo = &pExpr[idx]; + + // ts_comp column required the tag value for join filter + if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { + continue; + } + + // todo use tag column index to optimize performance + doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type, + pLocalExprInfo->base.resSchema.bytes); + + if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) + || pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_BOOL + || pLocalExprInfo->base.resSchema.type == TSDB_DATA_TYPE_TIMESTAMP) { + memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i, pLocalExprInfo->base.resSchema.bytes); + } else { + if (pCtx[idx].tag.pz != NULL) { + memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); + } + } + + offset += pLocalExprInfo->base.resSchema.bytes; + } + + //todo : use index to avoid iterator all possible output columns + if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); + } + } + + // set the tsBuf start position before check each data block + if (pRuntimeEnv->pTsBuf != NULL) { + setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); + } +} + +void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { + SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; + pBlock->info.rows = 0; + + int32_t code = TSDB_CODE_SUCCESS; + while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + // all results in current group have been returned to client, try next group + if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { + assert(pGroupResInfo->index == 0); + if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { + return; + } + } + + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); + + // current data are all dumped to result buffer, clear it + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + cleanupGroupResInfo(pGroupResInfo); + if (!incNextGroup(pGroupResInfo)) { + break; + } + } + + // enough results in data buffer, return + if (pBlock->info.rows >= threshold) { + break; + } + } +} + +static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) { + if (pTableQueryInfo == NULL) { + return; + } + + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + + SWITCH_ORDER(pTableQueryInfo->cur.order); + pTableQueryInfo->cur.vgroupIndex = -1; + + // set the index to be the end slot of result rows array + SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } else { + pResultRowInfo->curPos = -1; + } +} + +static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + SArray *tableKeyGroup = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + updateTableQueryInfoForReverseScan(pCheckInfo); + + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle +// STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); +// pTableKeyInfo->lastKey = pCheckInfo->lastKey; +// +// assert(pCheckInfo->pTable == pTableKeyInfo->pTable); + } + } +} + +void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t i = 0; i < numOfOutput; ++i) { + SWITCH_ORDER(pCtx[i].order); + } +} + +int32_t initResultRow(SResultRow *pResultRow) { + pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); + pResultRow->pageId = -1; + pResultRow->offset = -1; + return TSDB_CODE_SUCCESS; +} + +/* + * The start of each column SResultRowEntryInfo is denote by RowCellInfoOffset. + * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. + * +------------+-----------------result column 1-----------+-----------------result column 2-----------+ + * + SResultRow | SResultRowEntryInfo | intermediate buffer1 | SResultRowEntryInfo | intermediate buffer 2| + * +------------+-------------------------------------------+-------------------------------------------+ + * offset[0] offset[1] offset[2] + */ +void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { + SQLFunctionCtx* pCtx = pInfo->pCtx; + SSDataBlock* pDataBlock = pInfo->pRes; + int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; + + int64_t tid = 0; + pRuntimeEnv->keyBuf = realloc(pRuntimeEnv->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); + SResultRow* pRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid); + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); + + /* + * set the output buffer information and intermediate buffer + * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. + */ + struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); + cleanupResultRowEntry(pEntry); + + pCtx[i].resultInfo = pEntry; + pCtx[i].pOutput = pData->pData; + pCtx[i].currentStage = stage; + assert(pCtx[i].pOutput != NULL); + + // set the timestamp output buffer for top/bottom/diff query + int32_t fid = pCtx[i].functionId; + if (fid == FUNCTION_TOP || fid == FUNCTION_BOTTOM || fid == FUNCTION_DIFF || fid == FUNCTION_DERIVATIVE) { + if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + } + + initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); +} + +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows) { + SSDataBlock* pDataBlock = pBInfo->pRes; + + int32_t newSize = pDataBlock->info.rows + numOfInputRows + 5; // extra output buffer + if ((*bufCapacity) < newSize) { + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + + char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + if (p != NULL) { + pColInfo->pData = p; + + // it starts from the tail of the previously generated results. + pBInfo->pCtx[i].pOutput = pColInfo->pData; + (*bufCapacity) = newSize; + } else { + // longjmp + } + } + } + + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; + + // set the correct pointer after the memory buffer reallocated. + int32_t functionId = pBInfo->pCtx[i].functionId; + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { + if (i > 0) pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[i-1].pOutput; + } + } +} + +void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + bool needCopyTs = false; + int32_t tsNum = 0; + char *src = NULL; + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if (functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { + needCopyTs = true; + if (i > 0 && pCtx[i-1].functionId == FUNCTION_TS_DUMMY){ + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data + src = pColRes->pData; + } + }else if(functionId == FUNCTION_TS_DUMMY) { + tsNum++; + } + } + + if (!needCopyTs) return; + if (tsNum < 2) return; + if (src == NULL) return; + + for (int32_t i = 0; i < numOfOutput; i++) { + int32_t functionId = pCtx[i].functionId; + if(functionId == FUNCTION_TS_DUMMY) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); + memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); + } + } +} + +void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { + SSDataBlock* pDataBlock = pBInfo->pRes; + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + + int32_t functionId = pBInfo->pCtx[i].functionId; + if (functionId < 0) { + memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity)); + } + } +} + +void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { + for (int32_t j = 0; j < size; ++j) { + struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + if (isRowEntryInitialized(pResInfo)) { + continue; + } + +// if (pCtx[j].functionId < 0) { // todo udf initialization +// continue; +// } else { +// aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); +// } + } +} + +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { + if (status == QUERY_NOT_COMPLETED) { + pRuntimeEnv->status = status; + } else { + // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first + CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->status |= status; + } +} + +static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + if (pRuntimeEnv->pTsBuf) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } + + // reverse order time range + SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); + + SET_REVERSE_SCAN_FLAG(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + + switchCtxOrder(pCtx, numOfOutput); + SWITCH_ORDER(pQueryAttr->order.order); + + setupQueryRangeForReverseScan(pRuntimeEnv); +} + +void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfOutput = pOperator->numOfOutput; + if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { + // for each group result, call the finalize function for each column + if (pQueryAttr->groupbyColumn) { + closeAllResultRows(pResultRowInfo); + } + + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *buf = pResultRowInfo->pResult[i]; + if (!isResultRowClosed(pResultRowInfo, i)) { + continue; + } + + setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); + + for (int32_t j = 0; j < numOfOutput; ++j) { +// pCtx[j].startTs = buf->win.skey; +// if (pCtx[j].functionId < 0) { +// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); +// } else { +// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); +// } + } + + + /* + * set the number of output results for group by normal columns, the number of output rows usually is 1 except + * the top and bottom query + */ + buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); + } + + } else { + for (int32_t j = 0; j < numOfOutput; ++j) { +// if (pCtx[j].functionId < 0) { +// doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); +// } else { +// aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); +// } + } + } +} + +static bool hasMainOutput(SQueryAttr *pQueryAttr) { + for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + + if (functionId != FUNCTION_TS && functionId != FUNCTION_TAG && functionId != FUNCTION_TAGPRJ) { + return true; + } + } + + return false; +} + +STableQueryInfo *createTableQueryInfo(SQueryAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { + STableQueryInfo *pTableQueryInfo = buf; + + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; + + pTableQueryInfo->pTable = pTable; + pTableQueryInfo->cur.vgroupIndex = -1; + + // set more initial size of interval/groupby query + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr) || groupbyColumn) { + int32_t initialSize = 128; + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + } else { // in other aggregate query, do not initialize the windowResInfo + } + + return pTableQueryInfo; +} + +STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { + STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); + + pTableQueryInfo->win = win; + pTableQueryInfo->lastKey = win.skey; + + pTableQueryInfo->pTable = NULL; + pTableQueryInfo->cur.vgroupIndex = -1; + + // set more initial size of interval/groupby query + int32_t initialSize = 16; + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + if (code != TSDB_CODE_SUCCESS) { + tfree(pTableQueryInfo); + return NULL; + } + + return pTableQueryInfo; +} + +void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { + if (pTableQueryInfo == NULL) { + return; + } + + taosVariantDestroy(&pTableQueryInfo->tag); + cleanupResultRowInfo(&pTableQueryInfo->resInfo); +} + +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + + int32_t offset = 0; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); + + struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; + if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { + offset += pCtx[i].outputBytes; + continue; + } + + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, bufPage, pResult->offset, offset); + offset += pCtx[i].outputBytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId < 0) { + continue; + } + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + +// if (!pResInfo->initialized) { +// aAggs[functionId].init(&pCtx[i], pResInfo); +// } + } +} + +void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, + int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { + // for simple group by query without interval, all the tables belong to one group result. + int64_t uid = 0; + int64_t tid = 0; + + SResultRow* pResultRow = + doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid); + assert (pResultRow != NULL); + + /* + * not assign result buffer yet, add new result buffer + * all group belong to one result set, and each group result has different group id so set the id to be one + */ + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, tableGroupId, pRuntimeEnv->pQueryAttr->resultRowSize); + if (ret != TSDB_CODE_SUCCESS) { + return; + } + } + + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); +} + +void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, + TSKEY nextKey) { + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; + + // lastKey needs to be updated + pTableQueryInfo->lastKey = nextKey; + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == tableGroupId) { + return; + } + + doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, tableGroupId); + + // record the current active group id + pRuntimeEnv->prevGroupId = tableGroupId; +} + +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + + int16_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset); + offset += pCtx[i].outputBytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { + if(i > 0) pCtx[i].ptsOutputBuf = pCtx[i-1].pOutput; + } + + /* + * set the output buffer information and intermediate buffer, + * not all queries require the interResultBuf, such as COUNT + */ + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); + } +} + +void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + SSqlExpr* pExpr = &pExprInfo->base; +// if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && +// (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) && +// (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) { +// assert(pExpr->numOfParams == 1); +// +// int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; +// SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); +// +// doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); +// +// int16_t tagType = pCtx[0].tag.nType; +// if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), +//// pExprInfo->base.param[0].i, pCtx[0].tag.pz); +// } else { +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), +//// pExprInfo->base.param[0].i, pCtx[0].tag.i); +// } +// } +} + +int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, SVariant* pTag, STableQueryInfo *pTableQueryInfo) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + assert(pRuntimeEnv->pTsBuf != NULL); + + // both the master and supplement scan needs to set the correct ts comp start position + if (pTableQueryInfo->cur.vgroupIndex == -1) { + taosVariantAssign(&pTableQueryInfo->tag, pTag); + + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQueryAttr->vgId, &pTableQueryInfo->tag); + + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem)) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + } else { + //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); + } + + return -1; + } + + // Keep the cursor info of current table + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } + + } else { + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } + } + + return 0; +} + +// TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. +void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +#if 0 + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfExprs = pQueryAttr->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo1 = &(pExprInfo[i]); + if (pExprInfo1->base.functionId != FUNCTION_STDDEV_DST) { + continue; + } + + SSqlExpr* pExpr = &pExprInfo1->base; + + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (pQueryAttr->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQueryAttr->tagLen) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pExpr->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } + } + } + } + } +#endif +} + +void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; +#if 0 + int32_t numOfExprs = pQueryAttr->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr1 = &pExpr[i].base; + if (pExpr1->functionId != FUNCTION_STDDEV_DST) { + continue; + } + + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pExpr1->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } + } + } + } + } +#endif +} + +/* + * There are two cases to handle: + * + * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including pQueryAttr->lastKey, + * pQueryAttr->window.skey, and pQueryAttr->eKey. + * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be + * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there + * is a previous result generated or not. + */ +void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; + SResultRowInfo *pResultRowInfo = &pTableQueryInfo->resInfo; + + if (pResultRowInfo->curPos != -1) { + return; + } + + pTableQueryInfo->win.skey = key; + STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey}; + + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + STimeWindow w = TSWINDOW_INITIALIZER; + + TSKEY sk = MIN(win.skey, win.ekey); + TSKEY ek = MAX(win.skey, win.ekey); + getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); + +// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { +// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { +// assert(win.ekey == pQueryAttr->window.ekey); +// } +// +// pResultRowInfo->prevSKey = w.skey; +// } + + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +} + +/** + * copyToOutputBuf support copy data in ascending/descending order + * For interval query of both super table and table, copy the data in ascending order, since the output results are + * ordered in SWindowResutl already. While handling the group by query for both table and super table, + * all group result are completed already. + * + * @param pQInfo + * @param result + */ + +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + int32_t numOfResult = pBlock->info.rows; // there are already exists result rows + + int32_t start = 0; + int32_t step = -1; + + //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); + assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); + + if (orderType == TSDB_ORDER_ASC) { + start = pGroupResInfo->index; + step = 1; + } else { // desc order copy all data + start = numOfRows - pGroupResInfo->index - 1; + step = -1; + } + + for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { + SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + continue; + } + + int32_t numOfRowsToCopy = pRow->numOfRows; + if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { + break; + } + + pGroupResInfo->index += 1; + + SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); + + int32_t offset = 0; + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); + int32_t bytes = pColInfoData->info.bytes; + + char *out = pColInfoData->pData + numOfResult * bytes; + char *in = getPosInResultPage(pQueryAttr, page, pRow->offset, offset); + memcpy(out, in, bytes * numOfRowsToCopy); + + offset += bytes; + } + + numOfResult += numOfRowsToCopy; + if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full + break; + } + } + + //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + pBlock->info.rows = numOfResult; + return 0; +} + +static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { + assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + + pBlock->info.rows = 0; + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + return; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t orderType = (pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); + + // refactor : extract method + SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + //add condition (pBlock->info.rows >= 1) just to runtime happy + if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pBlock->info.rows >= 1) { + STimeWindow* w = &pBlock->info.window; + w->skey = *(int64_t*)pInfoData->pData; + w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); + } +} + +static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, + SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + // update the number of result for each, only update the number of rows for the corresponding window result. + if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { + return; + } + + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; + + for (int32_t j = 0; j < numOfOutput; ++j) { + int32_t functionId = pCtx[j].functionId; + if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { + continue; + } + +// SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); +// pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); + } + } +} + +static int32_t compressQueryColData(SColumnInfoData *pColRes, int32_t numOfRows, char *data, int8_t compressed) { + int32_t colSize = pColRes->info.bytes * numOfRows; + return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, + colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); +} + +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data, int8_t compressed, int32_t *compLen) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SSDataBlock* pRes = pRuntimeEnv->outputBuf; + + int32_t *compSizes = NULL; + int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; + + if (compressed) { + compSizes = calloc(numOfCols, sizeof(int32_t)); + } + + if (pQueryAttr->pExpr2 == NULL) { + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + if (compressed) { + compSizes[col] = compressQueryColData(pColRes, pRes->info.rows, data, compressed); + data += compSizes[col]; + *compLen += compSizes[col]; + compSizes[col] = htonl(compSizes[col]); + } else { + memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); + data += pColRes->info.bytes * pRes->info.rows; + } + } + } else { + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + if (compressed) { + compSizes[col] = htonl(compressQueryColData(pColRes, numOfRows, data, compressed)); + data += compSizes[col]; + *compLen += compSizes[col]; + compSizes[col] = htonl(compSizes[col]); + } else { + memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); + data += pColRes->info.bytes * numOfRows; + } + } + } + + if (compressed) { + memmove(data, (char *)compSizes, numOfCols * sizeof(int32_t)); + data += numOfCols * sizeof(int32_t); + + tfree(compSizes); + } + + int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); + *(int32_t*)data = htonl(numOfTables); + data += sizeof(int32_t); + + int32_t total = 0; + STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); + + while(item) { + STableIdInfo* pDst = (STableIdInfo*)data; + pDst->uid = htobe64(item->uid); + pDst->key = htobe64(item->key); + + data += sizeof(STableIdInfo); + total++; + + //qDebug("QInfo:0x%"PRIx64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key); + item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); + } + + //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); + + // Check if query is completed or not for stable query or normal table query respectively. + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } +} + +int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity, void** p) { +// for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { +// SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); +// p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows); +// } + + int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity - pOutput->info.rows); + pOutput->info.rows += numOfRows; + + return pOutput->info.rows; +} + +void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType) { + SQueryProfEvent event = {0}; + + event.eventType = eventType; + event.eventTime = taosGetTimestampUs(); + event.operatorType = operatorInfo->operatorType; + + if (operatorInfo->pRuntimeEnv) { + SQInfo* pQInfo = operatorInfo->pRuntimeEnv->qinfo; + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } + } +} + +void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { + SQueryProfEvent event; + event.eventType = QUERY_PROF_QUERY_ABORT; + event.eventTime = taosGetTimestampUs(); + event.abortCode = code; + + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } +} + +typedef struct { + uint8_t operatorType; + int64_t beginTime; + int64_t endTime; + int64_t selfTime; + int64_t descendantsTime; +} SOperatorStackItem; + +static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* event, SArray* opStack, SHashObj* profResults) { + item->endTime = event->eventTime; + item->selfTime = (item->endTime - item->beginTime) - (item->descendantsTime); + + for (int32_t j = 0; j < taosArrayGetSize(opStack); ++j) { + SOperatorStackItem* ancestor = taosArrayGet(opStack, j); + ancestor->descendantsTime += item->selfTime; + } + + uint8_t operatorType = item->operatorType; + SOperatorProfResult* result = taosHashGet(profResults, &operatorType, sizeof(operatorType)); + if (result != NULL) { + result->sumRunTimes++; + result->sumSelfTime += item->selfTime; + } else { + SOperatorProfResult opResult; + opResult.operatorType = operatorType; + opResult.sumSelfTime = item->selfTime; + opResult.sumRunTimes = 1; + taosHashPut(profResults, &(operatorType), sizeof(operatorType), + &opResult, sizeof(opResult)); + } +} + +void calculateOperatorProfResults(SQInfo* pQInfo) { + if (pQInfo->summary.queryProfEvents == NULL) { + //qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); + return; + } + + if (pQInfo->summary.operatorProfResults == NULL) { + //qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); + return; + } + + SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); + if (opStack == NULL) { + return; + } + + size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents); + SHashObj* profResults = pQInfo->summary.operatorProfResults; + + for (int i = 0; i < size; ++i) { + SQueryProfEvent* event = taosArrayGet(pQInfo->summary.queryProfEvents, i); + if (event->eventType == QUERY_PROF_BEFORE_OPERATOR_EXEC) { + SOperatorStackItem opItem; + opItem.operatorType = event->operatorType; + opItem.beginTime = event->eventTime; + opItem.descendantsTime = 0; + taosArrayPush(opStack, &opItem); + } else if (event->eventType == QUERY_PROF_AFTER_OPERATOR_EXEC) { + SOperatorStackItem* item = taosArrayPop(opStack); + assert(item->operatorType == event->operatorType); + doOperatorExecProfOnce(item, event, opStack, profResults); + } else if (event->eventType == QUERY_PROF_QUERY_ABORT) { + SOperatorStackItem* item; + while ((item = taosArrayPop(opStack)) != NULL) { + doOperatorExecProfOnce(item, event, opStack, profResults); + } + } + } + + taosArrayDestroy(opStack); +} + +void queryCostStatis(SQInfo *pQInfo) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryCostInfo *pSummary = &pQInfo->summary; + + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); + hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); + pSummary->hashSize = hashSize; + + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + + SResultRowPool* p = pQInfo->runtimeEnv.pool; + if (p != NULL) { + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + } else { + pSummary->winInfoSize = 0; + pSummary->numOfTimeWindows = 0; + } + + calculateOperatorProfResults(pQInfo); + + //qDebug("QInfo:0x%"PRIx64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " +// "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, +// pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, +// pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); + + //qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, +// pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); + + if (pSummary->operatorProfResults) { + SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); + while (opRes != NULL) { + //qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64, +// pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime); + opRes = taosHashIterate(pSummary->operatorProfResults, opRes); + } + } +} + +//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; +// +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); +// +// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed +// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; +// pQueryAttr->limit.offset = 0; +// return; +// } +// +// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; +// } else { +// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; +// } +// +// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); +// +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value +// TSKEY *keys = (TSKEY *) pColInfoData->pData; +// +// // update the offset value +// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; +// pQueryAttr->limit.offset = 0; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); +//} + +//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// +// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { +// return; +// } +// +// pQueryAttr->pos = 0; +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); +// +// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; +// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pQueryHandle)) { +// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); +// } +// +// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// +// if (pQueryAttr->limit.offset > blockInfo.rows) { +// pQueryAttr->limit.offset -= blockInfo.rows; +// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; +// pTableQueryInfo->lastKey += step; +// +// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// pQuery->limit.offset); +// } else { // find the appropriated start position in current block +// updateOffsetVal(pRuntimeEnv, &blockInfo); +// break; +// } +// } +// +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +//} + +//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// +// assert(pQueryAttr->limit.offset == 0); +// STimeWindow tw = *win; +// getNextTimeWindow(pQueryAttr, &tw); +// +// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || +// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { +// +// // load the data block and check data remaining in current data block +// // TODO optimize performance +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// tw = *win; +// int32_t startPos = +// getNextQualifiedWindow(pQueryAttr, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQueryAttr->pos = startPos; +// +// // reset the query start timestamp +// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pQueryAttr->window.skey = pTableQueryInfo->win.skey; +// TSKEY key = pTableQueryInfo->win.skey; +// +// pWindowResInfo->prevSKey = tw.skey; +// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index +// +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, +// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// pQueryAttr->current->lastKey); +// +// return key; +// } else { // do nothing +// pQueryAttr->window.skey = tw.skey; +// pWindowResInfo->prevSKey = tw.skey; +// pTableQueryInfo->lastKey = tw.skey; +// +// return tw.skey; +// } +// +// return true; +//} + +//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { +// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +// assert(*start <= pRuntimeEnv->current->lastKey); +// } else { +// assert(*start >= pRuntimeEnv->current->lastKey); +// } +// +// // if queried with value filter, do NOT forward query start position +// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { +// return true; +// } +// +// /* +// * 1. for interval without interpolation query we forward pQueryAttr->interval.interval at a time for +// * pQueryAttr->limit.offset times. Since hole exists, pQueryAttr->interval.interval*pQueryAttr->limit.offset value is +// * not valid. otherwise, we only forward pQueryAttr->limit.offset number of points +// */ +// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); +// +// STimeWindow w = TSWINDOW_INITIALIZER; +// bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); +// +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// +// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { +// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { +// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.skey, blockInfo.window.skey, pQueryAttr->window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// } else { +// getAlignQueryTimeWindow(pQueryAttr, blockInfo.window.ekey, pQueryAttr->window.ekey, blockInfo.window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// +// // the first time window +// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQueryAttr); +// +// while (pQueryAttr->limit.offset > 0) { +// STimeWindow tw = win; +// +// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { +// pQueryAttr->limit.offset -= 1; +// pWindowResInfo->prevSKey = win.skey; +// +// // current time window is aligned with blockInfo.window.ekey +// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; +// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { +// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; +// } +// } +// +// if (pQueryAttr->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } +// +// // current window does not ended in current data block, try next data block +// getNextTimeWindow(pQueryAttr, &tw); +// +// /* +// * If the next time window still starts from current data block, +// * load the primary timestamp column first, and then find the start position for the next queried time window. +// * Note that only the primary timestamp column is required. +// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required +// * time window resides in current data block. +// */ +// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { +// +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { +// pQueryAttr->limit.offset -= 1; +// } +// +// if (pQueryAttr->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } else { +// tw = win; +// int32_t startPos = +// getNextQualifiedWindow(pQueryAttr, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQueryAttr->pos = startPos; +// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pWindowResInfo->prevSKey = tw.skey; +// win = tw; +// } +// } else { +// break; // offset is not 0, and next time window begins or ends in the next block. +// } +// } +// } +// +// // check for error +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +// +// return true; +//} + +void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { + if (p->upstream == NULL) { + assert(p->numOfUpstream == 0); + } + + p->upstream = realloc(p->upstream, POINTER_BYTES * (p->numOfUpstream + 1)); + p->upstream[p->numOfUpstream++] = pUpstream; +} + +static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); + +static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; +#if 0 + // TODO set the tags scan handle + if (onlyQueryTags(pQueryAttr)) { + return TSDB_CODE_SUCCESS; + } + + STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); + if (pQueryAttr->tsCompQuery || pQueryAttr->pointInterpQuery) { + cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER; + } + + if (!isSTableQuery + && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1) + && (cond.order == TSDB_ORDER_ASC) + && (!QUERY_IS_INTERVAL_QUERY(pQueryAttr)) + && (!pQueryAttr->groupbyColumn) + && (!pQueryAttr->simpleAgg) + ) { + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); + cond.twindow = pCheckInfo->win; + } + + terrno = TSDB_CODE_SUCCESS; + if (isFirstLastRowQuery(pQueryAttr)) { + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + + // update the query time window + pQueryAttr->window = cond.twindow; + if (pQueryAttr->tableGroupInfo.numOfTables == 0) { + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; + } else { + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQueryAttr->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; + } + } + } + } else if (isCachedLastQuery(pQueryAttr)) { + pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + } else if (pQueryAttr->pointInterpQuery) { + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + } else { + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + } +#endif + return terrno; +} + +int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr, int32_t tbScanner, SArray* pOperator, + void* param) { + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + + SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + pQueryAttr->tsdb = tsdb; + + if (tsdb != NULL) { + int32_t code = setupQueryHandle(tsdb, pRuntimeEnv, pQInfo->qId, pQueryAttr->stableQuery); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr); + + pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (pQueryAttr->stableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0); + pRuntimeEnv->enableGroupData = false; + + pRuntimeEnv->pQueryAttr = pQueryAttr; + pRuntimeEnv->pTsBuf = pTsBuf; + pRuntimeEnv->cur.vgroupIndex = -1; + setResultBufSize(pQueryAttr, &pRuntimeEnv->resultInfo); + + switch(tbScanner) { + case OP_TableBlockInfoScan: { + pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + break; + } + case OP_TableSeqScan: { + pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + break; + } + case OP_DataBlocksOptScan: { + pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); + break; + } + case OP_TableScan: { + pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); + break; + } + default: { // do nothing + break; + } + } + + if (sourceOptr != NULL) { + assert(pRuntimeEnv->proot == NULL); + pRuntimeEnv->proot = sourceOptr; + } + + if (pTsBuf != NULL) { + int16_t order = (pQueryAttr->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); + } + + int32_t ps = DEFAULT_PAGE_SIZE; + getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); + + int32_t TENMB = 1024*1024*10; + int32_t code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // create runtime environment + int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; + pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); + pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); + if (pQInfo->summary.queryProfEvents == NULL) { + //qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); + } + + pQInfo->summary.operatorProfResults = + taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); + + if (pQInfo->summary.operatorProfResults == NULL) { + //qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); + } + + code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + return TSDB_CODE_SUCCESS; +} + +static void doTableQueryInfoTimeWindowCheck(SQueryAttr* pQueryAttr, STableQueryInfo* pTableQueryInfo) { + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + assert( + (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey >= pQueryAttr->window.skey && pTableQueryInfo->win.ekey <= pQueryAttr->window.ekey)); + } else { + assert( + (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey <= pQueryAttr->window.skey && pTableQueryInfo->win.ekey >= pQueryAttr->window.ekey)); + } +} + +//STsdbQueryCond createTsdbQueryCond(SQueryAttr* pQueryAttr, STimeWindow* win) { +// STsdbQueryCond cond = { +// .colList = pQueryAttr->tableCols, +// .order = pQueryAttr->order.order, +// .numOfCols = pQueryAttr->numOfCols, +// .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, +// .loadExternalRows = false, +// }; +// +// TIME_WINDOW_COPY(cond.twindow, *win); +// return cond; +//} + +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { + STableIdInfo tidInfo; +// STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); +// +// tidInfo.uid = id->uid; +// tidInfo.tid = id->tid; +// tidInfo.key = pTableQueryInfo->lastKey; + + return tidInfo; +} + +//static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) { +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); +// pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step; +// +// if (pTableQueryInfo->pTable == NULL) { +// return; +// } +// +// STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo); +// STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); +// if (idinfo != NULL) { +// assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); +// idinfo->key = tidInfo.key; +// } else { +// taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); +// } +//} + +static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); + + size_t num = taosArrayGetSize(group); + for (int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(group, j); + closeAllResultRows(&item->resInfo); + } + } +} + +static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*) param; + + STableScanInfo *pTableScanInfo = pOperator->info; + SSDataBlock *pBlock = &pTableScanInfo->block; + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + STableGroupInfo *pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; + + *newgroup = false; +#if 0 + while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { + longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + + pTableScanInfo->numOfBlocks += 1; + tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + + // todo opt + if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { + STableQueryInfo** pTableQueryInfo = + (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); + if (pTableQueryInfo == NULL) { + break; + } + + pRuntimeEnv->current = *pTableQueryInfo; + doTableQueryInfoTimeWindowCheck(pQueryAttr, *pTableQueryInfo); + + if (pRuntimeEnv->enableGroupData) { + if(pTableScanInfo->prevGroupId != -1 && pTableScanInfo->prevGroupId != (*pTableQueryInfo)->groupIndex) { + *newgroup = true; + } + } + + pTableScanInfo->prevGroupId = (*pTableQueryInfo)->groupIndex; + } + + // this function never returns error? + uint32_t status; + int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pOperator->pRuntimeEnv->env, code); + } + + // current block is ignored according to filter result by block statistics data, continue load the next block + if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { + continue; + } + + return pBlock; + } +#endif + + return NULL; +} + +static SSDataBlock* doTableScan(void* param, bool *newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + STableScanInfo *pTableScanInfo = pOperator->info; + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; + *newgroup = false; + + while (pTableScanInfo->current < pTableScanInfo->times) { + SSDataBlock* p = doTableScanImpl(pOperator, newgroup); + if (p != NULL) { + return p; + } + + if (++pTableScanInfo->current >= pTableScanInfo->times) { + if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { + return NULL; + } else { + break; + } + } + + // do prepare for the next round table scan operation +// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); +// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->scanFlag = REPEAT_SCAN; + + if (pRuntimeEnv->pTsBuf) { + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } + + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = 0; + } + + //qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, +// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + } + + SSDataBlock *p = NULL; + if (pTableScanInfo->reverseTimes > 0) { + setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); + +// STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); +// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + + //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, +// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + + pRuntimeEnv->scanFlag = REVERSE_SCAN; + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; +// pTableScanInfo->order = cond.order; + + if (pResultRowInfo->size > 0) { + pResultRowInfo->curPos = pResultRowInfo->size - 1; + } + + p = doTableScanImpl(pOperator, newgroup); + } + + return p; +} + +static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableScanInfo *pTableScanInfo = pOperator->info; + *newgroup = false; +#if 0 + STableBlockDist tableBlockDist = {0}; + tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + + int32_t numRowSteps = tsMaxRowsInFileBlock / TSDB_BLOCK_DIST_STEP_ROWS; + if (tsMaxRowsInFileBlock % TSDB_BLOCK_DIST_STEP_ROWS != 0) { + ++numRowSteps; + } + tableBlockDist.dataBlockInfos = taosArrayInit(numRowSteps, sizeof(SFileBlockInfo)); + taosArraySetSize(tableBlockDist.dataBlockInfos, numRowSteps); + tableBlockDist.maxRows = INT_MIN; + tableBlockDist.minRows = INT_MAX; + + tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + + SSDataBlock* pBlock = &pTableScanInfo->block; + pBlock->info.rows = 1; + pBlock->info.numOfCols = 1; + + SBufferWriter bw = tbufInitWriter(NULL, false); + blockDistInfoToBinary(&tableBlockDist, &bw); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + + int32_t len = (int32_t) tbufTell(&bw); + pColInfo->pData = malloc(len + sizeof(int32_t)); + + *(int32_t*) pColInfo->pData = len; + memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); + + tbufCloseWriter(&bw); + + SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); + pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0); + + pOperator->status = OP_EXEC_DONE; + return pBlock; +#endif + +} + +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { + assert(repeatTime > 0); + + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->current = 0; +// pInfo->prevGroupId = -1; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableScanOperator"; + pOperator->operatorType = OP_TableScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScan; + + return pOperator; +} + +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = 1; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + pInfo->current = 0; + pInfo->prevGroupId = -1; + pRuntimeEnv->enableGroupData = true; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = OP_TableSeqScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScanImpl; + + return pOperator; +} + +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {{0}}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.bytes = 1024; + infoData.info.colId = 0; + taosArrayPush(pInfo->block.pDataBlock, &infoData); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableBlockInfoScanOperator"; + pOperator->operatorType = OP_TableBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; + pOperator->exec = doBlockInfoScan; + + return pOperator; +} + +void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { + assert(pTableScanInfo != NULL && pDownstream != NULL); + + pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr + pTableScanInfo->numOfOutput = pDownstream->numOfOutput; + + if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { + SAggOperatorInfo* pAggInfo = pDownstream->info; + + pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_TimeWindow || pDownstream->operatorType == OP_AllTimeWindow) { + STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; + + pTableScanInfo->pCtx = pIntervalInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_Groupby) { + SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; + + pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_MultiTableTimeInterval || pDownstream->operatorType == OP_AllMultiTableTimeInterval) { + STableIntervalOperatorInfo *pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_Project) { + SProjectOperatorInfo *pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_SessionWindow) { + SSWindowOperatorInfo* pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_StateWindow) { + SStateWindowOperatorInfo* pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else { + assert(0); + } +} + +SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { + assert(repeatTime > 0); + + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->current = 0; + pInfo->order = pRuntimeEnv->pQueryAttr->order.order; + + SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); + pOptr->name = "DataBlocksOptimizedScanOperator"; + pOptr->operatorType = OP_DataBlocksOptScan; + pOptr->pRuntimeEnv = pRuntimeEnv; + pOptr->blockingOptr = false; + pOptr->info = pInfo; + pOptr->exec = doTableScan; + + return pOptr; +} + +SArray* getOrderCheckColumns(SQueryAttr* pQuery) { + int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0: taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); + + SArray* pOrderColumns = NULL; + if (numOfCols > 0) { + pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); + } else { + pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); + } + + if (pQuery->interval.interval > 0) { + if (pOrderColumns == NULL) { + pOrderColumns = taosArrayInit(1, sizeof(SColIndex)); + } + + SColIndex colIndex = {.colIndex = 0, .colId = 0, .flag = TSDB_COL_NORMAL}; + taosArrayPush(pOrderColumns, &colIndex); + } + + { + numOfCols = (int32_t) taosArrayGetSize(pOrderColumns); + for(int32_t i = 0; i < numOfCols; ++i) { + SColIndex* index = taosArrayGet(pOrderColumns, i); + for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + SSqlExpr* pExpr = &pQuery->pExpr1[j].base; + int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); + + if (index->colId == pExpr->colInfo.colId && + (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) { + index->colIndex = j; + index->colId = pExpr->resSchema.colId; + } + } + } + } + + return pOrderColumns; +} + +SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { + int32_t numOfCols = (pQuery->pGroupbyExpr == NULL)? 0 : taosArrayGetSize(pQuery->pGroupbyExpr->columnInfo); + + SArray* pOrderColumns = NULL; + if (numOfCols > 0) { + pOrderColumns = taosArrayDup(pQuery->pGroupbyExpr->columnInfo); + } else { + pOrderColumns = taosArrayInit(4, sizeof(SColIndex)); + } + + for(int32_t i = 0; i < numOfCols; ++i) { + SColIndex* index = taosArrayGet(pOrderColumns, i); + + bool found = false; + for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + SSqlExpr* pExpr = &pQuery->pExpr1[j].base; + int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); + + // FUNCTION_TAG_DUMMY function needs to be ignored + if (index->colId == pExpr->colInfo.colId && + ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && functionId == FUNCTION_TAG) || + (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && functionId == FUNCTION_PRJ))) { + index->colIndex = j; + index->colId = pExpr->resSchema.colId; + found = true; + break; + } + } + + assert(found && index->colIndex >= 0 && index->colIndex < pQuery->numOfOutput); + } + + return pOrderColumns; +} + +static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { + SMultiwayMergeInfo *pInfo = (SMultiwayMergeInfo*) param; + destroyBasicOperatorInfo(&pInfo->binfo, numOfOutput); + + taosArrayDestroy(pInfo->orderColumnList); + taosArrayDestroy(pInfo->groupColumnList); + tfree(pInfo->prevRow); + tfree(pInfo->currentGroupColData); +} +static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { + SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; + taosArrayDestroy(pInfo->orderColumnList); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); + tfree(pInfo->prevRow); +} + +SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, + SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { + SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); + + pInfo->resultRowFactor = + (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); + + pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx + + pInfo->multiGroupResults = groupResultMixedUp; + pInfo->pMerge = param; + pInfo->bufCapacity = 4096; + pInfo->udfInfo = pUdfInfo; + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); + pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); + + // TODO refactor + int32_t len = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { +// len += pExpr[i].base.; + } + + int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; + pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + int32_t offset = POINTER_BYTES * numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + + SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[index->colIndex].base.resSchema.bytes; + } + + numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; + pInfo->currentGroupColData = calloc(1, (POINTER_BYTES * numOfCols + len)); + offset = POINTER_BYTES * numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; + + SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); + offset += pExpr[index->colIndex].base.resSchema.bytes; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->seed = rand(); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "GlobalAggregate"; + pOperator->operatorType = OP_GlobalAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doGlobalAggregate; + pOperator->cleanup = destroyGlobalAggOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo *createMultiwaySortOperatorInfo(SQueryRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, + int32_t numOfRows, void *merger) { + SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); + + pInfo->pMerge = merger; + pInfo->bufCapacity = numOfRows; + pInfo->orderColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); + + { // todo extract method to create prev compare buffer + int32_t len = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { +// len += pExpr[i].base.colBytes; + } + + int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; + pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + + int32_t offset = POINTER_BYTES * numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + + SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); +// offset += pExpr[index->colIndex].base.colBytes; + } + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiwaySortOperator"; + pOperator->operatorType = OP_MultiwayMergeSort; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExpr; + pOperator->exec = doMultiwayMergeSort; + pOperator->cleanup = destroyGlobalAggOperatorInfo; + return pOperator; +} + +static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { + assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); + + int32_t numOfCols = pSrc->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); + + int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; + char* tmp = realloc(pCol2->pData, newSize); + if (tmp != NULL) { + pCol2->pData = tmp; + int32_t offset = pCol2->info.bytes * pDest->info.rows; + memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); + } else { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + } + + pDest->info.rows += pSrc->info.rows; + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doSort(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SOrderOperatorInfo* pInfo = pOperator->info; + + SSDataBlock* pBlock = NULL; + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + // start to flush data into disk and try do multiway merge sort + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + + int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); + if (code != TSDB_CODE_SUCCESS) { + // todo handle error + } + } + + int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; + void** pCols = calloc(numOfCols, POINTER_BYTES); + SSchema* pSchema = calloc(numOfCols, sizeof(SSchema)); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p1 = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); + pCols[i] = p1->pData; + pSchema[i].colId = p1->info.colId; + pSchema[i].bytes = p1->info.bytes; + pSchema[i].type = (uint8_t) p1->info.type; + } + + __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); +// taosqsort(pCols, pSchema, numOfCols, pInfo->pDataBlock->info.rows, pInfo->colIndex, comp); + + tfree(pCols); + tfree(pSchema); + return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; +} + +SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { + SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); + + { + SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); + pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData col = {{0}}; + col.info.colId = pExpr[i].base.colInfo.colId; +// col.info.bytes = pExpr[i].base.colBytes; +// col.info.type = pExpr[i].base.colType; + taosArrayPush(pDataBlock->pDataBlock, &col); + +// if (col.info.colId == pOrderVal->orderColId) { +// pInfo->colIndex = i; +// } + } + + pDataBlock->info.numOfCols = numOfOutput; +// pInfo->order = pOrderVal->order; + pInfo->pDataBlock = pDataBlock; + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "InMemoryOrder"; + pOperator->operatorType = OP_Order; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doSort; + pOperator->cleanup = destroyOrderOperatorInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { + return pTableScanInfo->order; +} + +// this is a blocking operator +static SSDataBlock* doAggregate(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + if (pRuntimeEnv->current != NULL) { + setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + } + + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + } + + doSetOperatorCompleted(pOperator); + + finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); + pInfo->pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); + + return pInfo->pRes; +} + +static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); + + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + + TSKEY key = 0; + if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + key = pBlock->info.window.ekey; + TSKEY_MAX_ADD(key, 1); + } else { + key = pBlock->info.window.skey; + TSKEY_MIN_SUB(key, -1); + } + + setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pRuntimeEnv->current->groupIndex, key); + doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->resultRowInfo); + + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, + pInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); + + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pInfo->pRes; +} + +static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + SProjectOperatorInfo* pProjectInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + SOptrBasicInfo *pInfo = &pProjectInfo->binfo; + + SSDataBlock* pRes = pInfo->pRes; + int32_t order = pRuntimeEnv->pQueryAttr->order.order; + + pRes->info.rows = 0; + + if (pProjectInfo->existDataBlock) { // TODO refactor + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + SSDataBlock* pBlock = pProjectInfo->existDataBlock; + pProjectInfo->existDataBlock = NULL; + *newgroup = true; + + // todo dynamic set tags + if (pTableQueryInfo != NULL) { + setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + + pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); + if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); + resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); + return pRes; + } + } + + while(1) { + bool prevVal = *newgroup; + + // The upstream exec may change the value of the newgroup, so use a local variable instead. + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + assert(*newgroup == false); + + *newgroup = prevVal; + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + break; + } + + // Return result of the previous group in the firstly. + if (*newgroup) { + if (pRes->info.rows > 0) { + pProjectInfo->existDataBlock = pBlock; + break; + } else { // init output buffer for a new group data +// for (int32_t j = 0; j < pOperator->numOfOutput; ++j) { +// aAggs[pInfo->pCtx[j].functionId].xFinalize(&pInfo->pCtx[j]); +// } + initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfOutput); + } + } + + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + // todo dynamic set tags + if (pTableQueryInfo != NULL) { + setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); + + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); + if (pRes->info.rows >= 1000/*pRuntimeEnv->resultInfo.threshold*/) { + break; + } + } + copyTsColoum(pRes, pInfo->pCtx, pOperator->numOfOutput); + resetResultRowEntryResult(pInfo->pCtx, pOperator->numOfOutput); + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; +} + +static SSDataBlock* doLimit(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SLimitOperatorInfo* pInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + SSDataBlock* pBlock = NULL; + while (1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + return NULL; + } + + if (pRuntimeEnv->currentOffset == 0) { + break; + } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { + pRuntimeEnv->currentOffset -= pBlock->info.rows; + } else { + int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + pBlock->info.rows = remain; + + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColInfoData->info.bytes; + memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); + } + + pRuntimeEnv->currentOffset = 0; + break; + } + } + + if (pInfo->total + pBlock->info.rows >= pInfo->limit) { + pBlock->info.rows = (int32_t)(pInfo->limit - pInfo->total); + pInfo->total = pInfo->limit; + + doSetOperatorCompleted(pOperator); + } else { + pInfo->total += pBlock->info.rows; + } + + return pBlock; +} + +static SSDataBlock* doFilter(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo *)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SFilterOperatorInfo* pCondInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + while (1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock *pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + doSetFilterColumnInfo(pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock); + assert(pRuntimeEnv->pTsBuf == NULL); + filterRowsInDataBlock(pRuntimeEnv, pCondInfo->pFilterInfo, pCondInfo->numOfFilterCols, pBlock, true); + + if (pBlock->info.rows > 0) { + return pBlock; + } + } + + doSetOperatorCompleted(pOperator); + return NULL; +} + +static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; +} + +static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; +} + +static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + int64_t st = taosGetTimestampUs(); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); + } + + pOperator->status = OP_RES_TO_RETURN; + pQueryAttr->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; +} + +static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pIntervalInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; + + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); + } + + pOperator->status = OP_RES_TO_RETURN; + pQueryAttr->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + int64_t st = taosGetTimestampUs(); + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); + + return pIntervalInfo->pRes; +} + +static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->current; + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + + SColumnInfoData* pTsColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); + TSKEY* tsList = (TSKEY*)pTsColInfoData->pData; + if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) { + pInfo->reptScan = true; + tfree(pInfo->prevData); + } + + pInfo->numOfRows = 0; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + char* val = ((char*)pColInfoData->pData) + bytes * j; + if (isNull(val, type)) { + continue; + } + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); + memcpy(pInfo->prevData, val, bytes); + pInfo->numOfRows = 1; + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->start = j; + + } else if (memcmp(pInfo->prevData, val, bytes) == 0) { + pInfo->curWindow.ekey = tsList[j]; + pInfo->numOfRows += 1; + //pInfo->start = j; + if (j == 0 && pInfo->start != 0) { + pInfo->numOfRows = 1; + pInfo->start = 0; + } + } else { + SResultRow* pResult = NULL; + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); + + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + memcpy(pInfo->prevData, val, bytes); + pInfo->numOfRows = 1; + pInfo->start = j; + + } + } + + SResultRow* pResult = NULL; + + pInfo->curWindow.ekey = pInfo->curWindow.skey; + int32_t ret = setResultOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, pSDataBlock->info.uid, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); +} + +static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SStateWindowOperatorInfo* pWindowInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + SOperatorInfo* upstream = pOperator->upstream[0]; + while (1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + break; + } + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); + if (pWindowInfo->colIndex == -1) { + pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); + } + doStateWindowAggImpl(pOperator, pWindowInfo, pBlock); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; +} + +static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SSWindowOperatorInfo* pWindowInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; + + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes; + } + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + //pQueryAttr->order.order = TSDB_ORDER_ASC; + int32_t order = pQueryAttr->order.order; + STimeWindow win = pQueryAttr->window; + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order); + doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); + } + + // restore the value + pQueryAttr->order.order = order; + pQueryAttr->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); + + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; +} + +static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SGroupbyOperatorInfo *pInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->binfo.pRes; + } + + SOperatorInfo* upstream = pOperator->upstream[0]; + + while(1) { + publishOperatorProfEvent(upstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = upstream->exec(upstream, newgroup); + publishOperatorProfEvent(upstream, QUERY_PROF_AFTER_OPERATOR_EXEC); + if (pBlock == NULL) { + break; + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order); + setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + if (pInfo->colIndex == -1) { + pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); + } + + doHashGroupbyAgg(pOperator, pInfo, pBlock); + } + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->binfo.resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows + finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + } else { + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + } + + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); + if (!pRuntimeEnv->pQueryAttr->stableQuery) { + sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + } + + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } + + return pInfo->binfo.pRes; +} + +static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, SQueryRuntimeEnv* pRuntimeEnv, bool* newgroup) { + pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); + + taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->existNewGroupBlock); + + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + pInfo->existNewGroupBlock = NULL; + *newgroup = true; +} + +static void doHandleRemainBlockFromNewGroup(SFillOperatorInfo *pInfo, SQueryRuntimeEnv *pRuntimeEnv, bool *newgroup) { + if (taosFillHasMoreResults(pInfo->pFillInfo)) { + *newgroup = false; + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity, pInfo->p); + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult)) { + return; + } + } + + // handle the cached new group data block + if (pInfo->existNewGroupBlock) { + doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); + } +} + +static SSDataBlock* doFill(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + SFillOperatorInfo *pInfo = pOperator->info; + pInfo->pRes->info.rows = 0; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || (!pInfo->multigroupResult && pInfo->pRes->info.rows > 0)) { + return pInfo->pRes; + } + + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + SSDataBlock* pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (*newgroup) { + assert(pBlock != NULL); + } + + if (*newgroup && pInfo->totalInputRows > 0) { // there are already processed current group data block + pInfo->existNewGroupBlock = pBlock; + *newgroup = false; + + // Fill the previous group data block, before handle the data block of new group. + // Close the fill operation for previous group data block + taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + } else { + if (pBlock == NULL) { + if (pInfo->totalInputRows == 0) { + pOperator->status = OP_EXEC_DONE; + return NULL; + } + + taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQueryAttr->window.ekey); + } else { + pInfo->totalInputRows += pBlock->info.rows; + taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, pBlock->info.window.ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock); + } + } + + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity, pInfo->p); + + // current group has no more result to return + if (pInfo->pRes->info.rows > 0) { + // 1. The result in current group not reach the threshold of output result, continue + // 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL || (!pInfo->multigroupResult)) { + return pInfo->pRes; + } + + doHandleRemainBlockFromNewGroup(pInfo, pRuntimeEnv, newgroup); + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold || pBlock == NULL) { + return pInfo->pRes; + } + } else if (pInfo->existNewGroupBlock) { // try next group + assert(pBlock != NULL); + doHandleRemainBlockForNewGroupImpl(pInfo, pRuntimeEnv, newgroup); + + if (pInfo->pRes->info.rows > pRuntimeEnv->resultInfo.threshold) { + return pInfo->pRes; + } + } else { + return NULL; + } + } +} + +// todo set the attribute of query scan count +static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr) { + for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { + int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); + if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) { + return 2; + } + } + + return 1; +} + +static void destroyOperatorInfo(SOperatorInfo* pOperator) { + if (pOperator == NULL) { + return; + } + + if (pOperator->cleanup != NULL) { + pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + } + + if (pOperator->upstream != NULL) { + for(int32_t i = 0; i < pOperator->numOfUpstream; ++i) { + destroyOperatorInfo(pOperator->upstream[i]); + } + + tfree(pOperator->upstream); + pOperator->numOfUpstream = 0; + } + + tfree(pOperator->info); + tfree(pOperator); +} + +SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + int32_t numOfRows = (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->seed = rand(); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MASTER_SCAN); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableAggregate"; + pOperator->operatorType = OP_Aggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAggregate; + pOperator->cleanup = destroyAggOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { + assert(pInfo != NULL); + + destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); + tfree(pInfo->rowCellInfoOffset); + + cleanupResultRowInfo(&pInfo->resultRowInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { + SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; + doDestroyBasicInfo(pInfo, numOfOutput); +} +static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + tfree(pInfo->prevData); +} +static void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = (SAggOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} +static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { + SSWindowOperatorInfo* pInfo = (SSWindowOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { + SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; + pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); + tfree(pInfo->p); +} + +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + tfree(pInfo->prevData); +} + +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*) param; + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { + SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; + pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock); +} + +static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { + SFilterOperatorInfo* pInfo = (SFilterOperatorInfo*) param; + doDestroyFilterInfo(pInfo->pFilterInfo, pInfo->numOfFilterCols); +} + +static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { + SDistinctOperatorInfo* pInfo = (SDistinctOperatorInfo*) param; + taosHashCleanup(pInfo->pSet); + tfree(pInfo->buf); + taosArrayDestroy(pInfo->pDistinctDataInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + + size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableAggregate"; + pOperator->operatorType = OP_MultiTableAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableAggregate; + pOperator->cleanup = destroyAggOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); + + pInfo->seed = rand(); + pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; + + SOptrBasicInfo* pBInfo = &pInfo->binfo; + pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); + pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); + + initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "ProjectOperator"; + pOperator->operatorType = OP_Project; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doProjectOperation; + pOperator->cleanup = destroyProjectOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int32_t* numOfFilterCols) { +#if 0 + SColumnInfo* pCols = calloc(numOfOutput, sizeof(SColumnInfo)); + + int32_t numOfFilter = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { + if (pExpr[i].base.flist.numOfFilters > 0) { + numOfFilter += 1; + } + + pCols[i].type = pExpr[i].base.resSchema.type; + pCols[i].bytes = pExpr[i].base.resSchema.bytes; + pCols[i].colId = pExpr[i].base.resSchema.colId; + + pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters; + if (pCols[i].flist.numOfFilters != 0) { + pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo)); + memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo)); + } else { + // avoid runtime error + pCols[i].flist.filterInfo = NULL; + } + } + + assert(numOfFilter > 0); + + *numOfFilterCols = numOfFilter; + return pCols; +#endif + + return 0; +} + +SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter) { + SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); + + assert(numOfFilter > 0 && pCols != NULL); + doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); + pInfo->numOfFilterCols = numOfFilter; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "FilterOperator"; + pOperator->operatorType = OP_Filter; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExpr; + pOperator->exec = doFilter; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyConditionOperatorInfo; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { + SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); + pInfo->limit = pRuntimeEnv->pQueryAttr->limit.limit; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "LimitOperator"; + pOperator->operatorType = OP_Limit; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->exec = doLimit; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + appendUpstream(pOperator, upstream); + + return pOperator; +} + +SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = OP_TimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + + +SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "AllTimeIntervalAggOperator"; + pOperator->operatorType = OP_AllTimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doAllIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SStateWindowOperatorInfo* pInfo = calloc(1, sizeof(SStateWindowOperatorInfo)); + pInfo->colIndex = -1; + pInfo->reptScan = false; + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "StateWindowOperator"; + pOperator->operatorType = OP_StateWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doStateWindowAgg; + pOperator->cleanup = destroyStateWindowOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} +SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->prevTs = INT64_MIN; + pInfo->reptScan = false; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = OP_SessionWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doSessionWindowAgg; + pOperator->cleanup = destroySWindowOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_MultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "AllMultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_AllMultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAllSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + appendUpstream(pOperator, upstream); + + return pOperator; +} + + +SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); + pInfo->colIndex = -1; // group by column index + + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * + (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "GroupbyAggOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Groupby; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashGroupbyAggregate; + pOperator->cleanup = destroyGroupbyOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { + SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + pInfo->multigroupResult = multigroupResult; + + { + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); + STimeWindow w = TSWINDOW_INITIALIZER; + + TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); + TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); + getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); + + pInfo->pFillInfo = + taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, + pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, + (int8_t)pQueryAttr->precision, pQueryAttr->fillType, pColInfo, pRuntimeEnv->qinfo); + + pInfo->p = calloc(numOfOutput, POINTER_BYTES); + } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "FillOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Fill; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doFill; + pOperator->cleanup = destroySFillOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* pMerger, bool multigroupResult) { + SSLimitOperatorInfo* pInfo = calloc(1, sizeof(SSLimitOperatorInfo)); + + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + + pInfo->orderColumnList = getResultGroupCheckColumns(pQueryAttr); + pInfo->slimit = pQueryAttr->slimit; + pInfo->limit = pQueryAttr->limit; + pInfo->capacity = pRuntimeEnv->resultInfo.capacity; + pInfo->threshold = (int64_t)(pInfo->capacity * 0.8); + pInfo->currentOffset = pQueryAttr->limit.offset; + pInfo->currentGroupOffset = pQueryAttr->slimit.offset; + pInfo->multigroupResult= multigroupResult; + + // TODO refactor + int32_t len = 0; + for(int32_t i = 0; i < numOfOutput; ++i) { + len += pExpr[i].base.resSchema.bytes; + } + + int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; + pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + + int32_t offset = POINTER_BYTES * numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + + SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); + offset += pExpr[index->colIndex].base.resSchema.bytes; + } + + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "SLimitOperator"; + pOperator->operatorType = OP_SLimit; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->exec = doSLimit; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroySlimitOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +static SSDataBlock* doTagScan(void* param, bool* newgroup) { +#if 0 + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; + + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; + *newgroup = false; + + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); + if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id + SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + assert(pQueryAttr->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->base.resSchema.bytes; + + count = 0; + + int16_t bytes = pExprInfo->base.resSchema.bytes; + int16_t type = pExprInfo->base.resSchema.type; + + for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.colInfo.colId) { + bytes = pQueryAttr->tagColList[i].bytes; + type = pQueryAttr->tagColList[i].type; + break; + } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); + + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); + + *(int32_t *)output = id->tid; + output += sizeof(id->tid); + + *(int32_t *)output = pQueryAttr->vgId; + output += sizeof(pQueryAttr->vgId); + + char* data = NULL; + if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); + } + + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; + } + + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; + + pOperator->status = OP_EXEC_DONE; + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo + + count = 0; + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + + STableQueryInfo* item = taosArrayGetP(pa, i); + + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { + continue; + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].base.resSchema.type; + bytes = pExprInfo[j].base.resSchema.bytes; + + if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + } + + dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } + + count += 1; + } + + if (pInfo->curPos >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; + } + + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); + } + + if (pOperator->status == OP_EXEC_DONE) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + } + + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; + +#endif +} + +SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { + STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + assert(numOfGroup == 0 || numOfGroup == 1); + + pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + pInfo->curPos = 0; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "SeqTableTagScan"; + pOperator->operatorType = OP_TagScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyTagScanOperatorInfo; + + return pOperator; +} +static bool initMultiDistinctInfo(SDistinctOperatorInfo *pInfo, SOperatorInfo* pOperator, SSDataBlock *pBlock) { + if (taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput) { + // distinct info already inited + return true; + } + for (int i = 0; i < pOperator->numOfOutput; i++) { +// pInfo->totalBytes += pOperator->pExpr[i].base.colBytes; + } + for (int i = 0; i < pOperator->numOfOutput; i++) { + int numOfBlock = (int)(taosArrayGetSize(pBlock->pDataBlock)); + assert(i < numOfBlock); + for (int j = 0; j < numOfBlock; j++) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, j); + if (pColDataInfo->info.colId == pOperator->pExpr[i].base.resSchema.colId) { + SDistinctDataInfo item = {.index = j, .type = pColDataInfo->info.type, .bytes = pColDataInfo->info.bytes}; + taosArrayInsert(pInfo->pDistinctDataInfo, i, &item); + } + } + } + pInfo->totalBytes += (int32_t)strlen(MULTI_KEY_DELIM) * (pOperator->numOfOutput); + pInfo->buf = calloc(1, pInfo->totalBytes); + return taosArrayGetSize(pInfo->pDistinctDataInfo) == pOperator->numOfOutput ? true : false; +} + +static void buildMultiDistinctKey(SDistinctOperatorInfo *pInfo, SSDataBlock *pBlock, int32_t rowId) { + char *p = pInfo->buf; + memset(p, 0, pInfo->totalBytes); + + for (int i = 0; i < taosArrayGetSize(pInfo->pDistinctDataInfo); i++) { + SDistinctDataInfo* pDistDataInfo = (SDistinctDataInfo *)taosArrayGet(pInfo->pDistinctDataInfo, i); + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); + char *val = ((char *)pColDataInfo->pData) + pColDataInfo->info.bytes * rowId; + if (isNull(val, pDistDataInfo->type)) { + p += pDistDataInfo->bytes; + continue; + } + if (IS_VAR_DATA_TYPE(pDistDataInfo->type)) { + memcpy(p, varDataVal(val), varDataLen(val)); + p += varDataLen(val); + } else { + memcpy(p, val, pDistDataInfo->bytes); + p += pDistDataInfo->bytes; + } + memcpy(p, MULTI_KEY_DELIM, strlen(MULTI_KEY_DELIM)); + p += strlen(MULTI_KEY_DELIM); + } +} + +static SSDataBlock* hashDistinct(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SDistinctOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pRes = pInfo->pRes; + + pRes->info.rows = 0; + SSDataBlock* pBlock = NULL; + + while(1) { + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); + pBlock = pOperator->upstream[0]->exec(pOperator->upstream[0], newgroup); + publishOperatorProfEvent(pOperator->upstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + break; + } + if (!initMultiDistinctInfo(pInfo, pOperator, pBlock)) { + doSetOperatorCompleted(pOperator); + break; + } + // ensure result output buf + if (pRes->info.rows + pBlock->info.rows > pInfo->outputCapacity) { + int32_t newSize = pRes->info.rows + pBlock->info.rows; + for (int i = 0; i < taosArrayGetSize(pRes->pDataBlock); i++) { + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, i); + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, i); + char* tmp = realloc(pResultColInfoData->pData, newSize * pDistDataInfo->bytes); + if (tmp == NULL) { + return NULL; + } else { + pResultColInfoData->pData = tmp; + } + } + pInfo->outputCapacity = newSize; + } + + for (int32_t i = 0; i < pBlock->info.rows; i++) { + buildMultiDistinctKey(pInfo, pBlock, i); + if (taosHashGet(pInfo->pSet, pInfo->buf, pInfo->totalBytes) == NULL) { + int32_t dummy; + taosHashPut(pInfo->pSet, pInfo->buf, pInfo->totalBytes, &dummy, sizeof(dummy)); + for (int j = 0; j < taosArrayGetSize(pRes->pDataBlock); j++) { + SDistinctDataInfo* pDistDataInfo = taosArrayGet(pInfo->pDistinctDataInfo, j); // distinct meta info + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pDistDataInfo->index); //src + SColumnInfoData* pResultColInfoData = taosArrayGet(pRes->pDataBlock, j); // dist + + char* val = ((char*)pColInfoData->pData) + pDistDataInfo->bytes * i; + char *start = pResultColInfoData->pData + pDistDataInfo->bytes * pInfo->pRes->info.rows; + memcpy(start, val, pDistDataInfo->bytes); + } + pRes->info.rows += 1; + } + } + + if (pRes->info.rows >= pInfo->threshold) { + break; + } + } + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; +} + +SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SDistinctOperatorInfo* pInfo = calloc(1, sizeof(SDistinctOperatorInfo)); + pInfo->totalBytes = 0; + pInfo->buf = NULL; + pInfo->threshold = tsMaxNumOfDistinctResults; // distinct result threshold + pInfo->outputCapacity = 4096; + pInfo->pDistinctDataInfo = taosArrayInit(numOfOutput, sizeof(SDistinctDataInfo)); + pInfo->pSet = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) pInfo->outputCapacity); + + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "DistinctOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Distinct; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashDistinct; + pOperator->pExpr = pExpr; + pOperator->cleanup = destroyDistinctOperatorInfo; + + appendUpstream(pOperator, upstream); + return pOperator; +} + +static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { + int32_t j = 0; + + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + return TSDB_TBNAME_COLUMN_INDEX; + } + + while(j < pTableInfo->numOfTags) { + if (pExpr->colInfo.colId == pTagCols[j].colId) { + return j; + } + + j += 1; + } + + } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data + return TSDB_UD_COLUMN_INDEX; + } else { + while (j < pTableInfo->numOfCols) { + if (pExpr->colInfo.colId == pTableInfo->colList[j].colId) { + return j; + } + + j += 1; + } + } + + return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value +} + +bool validateExprColumnInfo(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { + int32_t j = getColumnIndexInSource(pTableInfo, pExpr, pTagCols); + return j != INT32_MIN; +} + +static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { + if (pQueryMsg->interval.interval < 0) { + //qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval); + return false; + } + + if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_ID) { + //qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap); + return false; + } + + if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) { + //qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg, +// pQueryMsg->sw.gap, pQueryMsg->interval.interval); + return false; + } + + if (pQueryMsg->numOfTables <= 0) { + //qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); + return false; + } + + if (pQueryMsg->numOfGroupCols < 0) { + //qError("qmsg:%p illegal value of numOfGroupbyCols %d", pQueryMsg, pQueryMsg->numOfGroupCols); + return false; + } + + if (pQueryMsg->numOfOutput > TSDB_MAX_COLUMNS || pQueryMsg->numOfOutput <= 0) { + //qError("qmsg:%p illegal value of output columns %d", pQueryMsg, pQueryMsg->numOfOutput); + return false; + } + + return true; +} + +static bool validateQueryTableCols(SQueriedTableInfo* pTableInfo, SSqlExpr** pExpr, int32_t numOfOutput, + SColumnInfo* pTagCols, void* pMsg) { + int32_t numOfTotal = pTableInfo->numOfCols + pTableInfo->numOfTags; + if (pTableInfo->numOfCols < 0 || pTableInfo->numOfTags < 0 || numOfTotal > TSDB_MAX_COLUMNS) { + //qError("qmsg:%p illegal value of numOfCols %d numOfTags:%d", pMsg, pTableInfo->numOfCols, pTableInfo->numOfTags); + return false; + } + + if (numOfTotal == 0) { // table total columns are not required. +// for(int32_t i = 0; i < numOfOutput; ++i) { +// SSqlExpr* p = pExpr[i]; +// if ((p->functionId == FUNCTION_TAGPRJ) || +// (p->functionId == FUNCTION_TID_TAG && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || +// (p->functionId == FUNCTION_COUNT && p->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || +// (p->functionId == FUNCTION_BLKINFO)) { +// continue; +// } +// +// return false; +// } + } + + for(int32_t i = 0; i < numOfOutput; ++i) { + if (!validateExprColumnInfo(pTableInfo, pExpr[i], pTagCols)) { + return TSDB_CODE_QRY_INVALID_MSG; + } + } + + return true; +} + +static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **pTableIdList) { + assert(pQueryMsg->numOfTables > 0); + + *pTableIdList = taosArrayInit(pQueryMsg->numOfTables, sizeof(STableIdInfo)); + + for (int32_t j = 0; j < pQueryMsg->numOfTables; ++j) { + STableIdInfo* pTableIdInfo = (STableIdInfo *)pMsg; + pTableIdInfo->uid = htobe64(pTableIdInfo->uid); + pTableIdInfo->key = htobe64(pTableIdInfo->key); + + taosArrayPush(*pTableIdList, pTableIdInfo); + pMsg += sizeof(STableIdInfo); + } + + return pMsg; +} + +static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { + for (int32_t f = 0; f < numOfFilters; ++f) { + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); + + SColumnFilterInfo *pColFilter = &pColFilters[f]; + pColFilter->filterstr = htons(pFilterMsg->filterstr); + + (*pMsg) += sizeof(SColumnFilterInfo); + + if (pColFilter->filterstr) { + pColFilter->len = htobe64(pFilterMsg->len); + + pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy((void *)pColFilter->pz, (*pMsg), (size_t)pColFilter->len); + (*pMsg) += (pColFilter->len + 1); + } else { + pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); + pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); + } + + pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); + pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * pQueryMsg->head has been converted before this function is called. + * + * @param pQueryMsg + * @param pTableIdList + * @param pExpr + * @return + */ +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { + int32_t code = TSDB_CODE_SUCCESS; + +// if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { +// return TSDB_CODE_QRY_INVALID_MSG; +// } + + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); + pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); + pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); + pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); + pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); + pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); + pQueryMsg->limit = htobe64(pQueryMsg->limit); + pQueryMsg->offset = htobe64(pQueryMsg->offset); + pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); + + pQueryMsg->order = htons(pQueryMsg->order); + pQueryMsg->orderColId = htons(pQueryMsg->orderColId); + pQueryMsg->queryType = htonl(pQueryMsg->queryType); +// pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); + + pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); + pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); + pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); + + pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); + pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); + + pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); + pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); + pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); + pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); + + pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); +// pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); + pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); + pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); + pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); + pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); + pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); + pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); + pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); + pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); + pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); + pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); + + // query msg safety check + if (!validateQueryMsg(pQueryMsg)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } + + char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; + for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { + SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; + + pColInfo->colId = htons(pColInfo->colId); + pColInfo->type = htons(pColInfo->type); + pColInfo->bytes = htons(pColInfo->bytes); + pColInfo->flist.numOfFilters = 0; + + if (!isValidDataType(pColInfo->type)) { + //qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } + +/* + int32_t numOfFilters = pColInfo->flist.numOfFilters; + if (numOfFilters > 0) { + pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->flist.filterInfo == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + } + + code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); + if (code != TSDB_CODE_SUCCESS) { + goto _cleanup; + } +*/ + } + + if (pQueryMsg->colCondLen > 0) { + param->colCond = calloc(1, pQueryMsg->colCondLen); + if (param->colCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); + pMsg += pQueryMsg->colCondLen; + } + + + param->tableScanOperator = pQueryMsg->tableScanOperator; + param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (param->pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; + + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + param->pExpr[i] = pExprMsg; + + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +// pExprMsg->colBytes = htons(pExprMsg->colBytes); +// pExprMsg->colType = htons(pExprMsg->colType); + +// pExprMsg->resType = htons(pExprMsg->resType); +// pExprMsg->resBytes = htons(pExprMsg->resBytes); + pExprMsg->interBytes = htonl(pExprMsg->interBytes); + +// pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); +// pExprMsg->resColId = htons(pExprMsg->resColId); +// pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); + pMsg += sizeof(SSqlExpr); + + for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { + pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); + pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); + + if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { + pExprMsg->param[j].pz = pMsg; + pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. + } else { + pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); + } + } + +// int16_t functionId = pExprMsg->functionId; +// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { +// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; +// } +// } + +// if (pExprMsg->flist.numOfFilters > 0) { +// pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); +// } +// +// deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); + pExprMsg = (SSqlExpr *)pMsg; + } + + if (pQueryMsg->secondStageOutput) { + pExprMsg = (SSqlExpr *)pMsg; + param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); + + for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { + param->pSecExpr[i] = pExprMsg; + + pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); + pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +// pExprMsg->resType = htons(pExprMsg->resType); +// pExprMsg->resBytes = htons(pExprMsg->resBytes); +// pExprMsg->colBytes = htons(pExprMsg->colBytes); +// pExprMsg->colType = htons(pExprMsg->colType); + +// pExprMsg->functionId = htons(pExprMsg->functionId); + pExprMsg->numOfParams = htons(pExprMsg->numOfParams); + + pMsg += sizeof(SSqlExpr); + + for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { + pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); + pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); + + if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { + pExprMsg->param[j].pz = pMsg; + pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. + } else { + pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); + } + } + +// int16_t functionId = pExprMsg->functionId; +// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { +// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. +// code = TSDB_CODE_QRY_INVALID_MSG; +// goto _cleanup; +// } +// } + + pExprMsg = (SSqlExpr *)pMsg; + } + } + + pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); + + if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns + param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + if (param->pGroupColIndex == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colId); + + param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colIndex); + + param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].flag); + + memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); + pMsg += tListLen(param->pGroupColIndex[i].name); + } + + pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); + pQueryMsg->orderType = htons(pQueryMsg->orderType); + } + + pQueryMsg->fillType = htons(pQueryMsg->fillType); + if (pQueryMsg->fillType != TSDB_FILL_NONE) { + pQueryMsg->fillVal = (uint64_t)(pMsg); + + int64_t *v = (int64_t *)pMsg; + for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { + v[i] = htobe64(v[i]); + } + + pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; + } + + if (pQueryMsg->numOfTags > 0) { + param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (param->pTagColumnInfo == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { + SColumnInfo* pTagCol = (SColumnInfo*) pMsg; + + pTagCol->colId = htons(pTagCol->colId); + pTagCol->bytes = htons(pTagCol->bytes); + pTagCol->type = htons(pTagCol->type); +// pTagCol->flist.numOfFilters = 0; + + param->pTagColumnInfo[i] = *pTagCol; + pMsg += sizeof(SColumnInfo); + } + } + + // the tag query condition expression string is located at the end of query msg + if (pQueryMsg->tagCondLen > 0) { + param->tagCond = calloc(1, pQueryMsg->tagCondLen); + if (param->tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); + pMsg += pQueryMsg->tagCondLen; + } + + if (pQueryMsg->prevResultLen > 0) { + param->prevResult = calloc(1, pQueryMsg->prevResultLen); + if (param->prevResult == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); + pMsg += pQueryMsg->prevResultLen; + } + +// if (pQueryMsg->tbnameCondLen > 0) { +// param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); +// if (param->tbnameCond == NULL) { +// code = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto _cleanup; +// } +// +// strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); +// pMsg += pQueryMsg->tbnameCondLen; +// } + + //skip ts buf + if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { + pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; + } + + param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); + for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { + int32_t op = htonl(*(int32_t*)pMsg); + taosArrayPush(param->pOperator, &op); + + pMsg += sizeof(int32_t); + } + + if (pQueryMsg->udfContentLen > 0) { + // todo extract udf function in tudf.c +// param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); +// param->pUdfInfo->contLen = pQueryMsg->udfContentLen; +// +// pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; +// param->pUdfInfo->resType = *(int8_t*) pMsg; +// pMsg += sizeof(int8_t); +// +// param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); +// pMsg += sizeof(int16_t); +// +// tstr* name = (tstr*)(pMsg); +// param->pUdfInfo->name = strndup(name->data, name->len); +// +// pMsg += varDataTLen(name); +// param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); +// pMsg += sizeof(int32_t); +// +// param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); +// pMsg += sizeof(int32_t); +// +// param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); +// memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); + + pMsg += pQueryMsg->udfContentLen; + } + + param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); + + SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; + if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { + code = TSDB_CODE_QRY_INVALID_MSG; + goto _cleanup; + } + + //qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " +// "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, +// pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, +// pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, +// pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); + + //qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); + return TSDB_CODE_SUCCESS; + +_cleanup: + freeParam(param); + return code; +} + +int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { + if (filterNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + *dst = calloc(filterNum, sizeof(*src)); + if (*dst == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(*dst, src, sizeof(*src) * filterNum); + + for (int32_t i = 0; i < filterNum; i++) { + if ((*dst)[i].filterstr && dst[i]->len > 0) { + void *pz = calloc(1, (size_t)(*dst)[i].len + 1); + + if (pz == NULL) { + if (i == 0) { + free(*dst); + } else { + freeColumnFilterInfo(*dst, i); + } + + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + memcpy(pz, (void *)src->pz, (size_t)src->len + 1); + + (*dst)[i].pz = (int64_t)pz; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { + //qDebug("qmsg:%p create arithmetic expr from binary", pQueryMsg); + + tExprNode* pExprNode = NULL; + TRY(TSDB_MAX_TAG_CONDITIONS) { + pExprNode = exprTreeFromBinary(pExprInfo->base.param[0].pz, pExprInfo->base.param[0].nLen); + } CATCH( code ) { + CLEANUP_EXECUTE(); + //qError("qmsg:%p failed to create arithmetic expression string from:%s, reason: %s", pQueryMsg, pExprInfo->base.param[0].pz, tstrerror(code)); + return code; + } END_TRY + + if (pExprNode == NULL) { + //qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pExprInfo->base.param[0].pz); + return TSDB_CODE_QRY_APP_ERROR; + } + + pExprInfo->pExpr = pExprNode; + return TSDB_CODE_SUCCESS; +} + + +static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { + for (int32_t i = 0; i < numOfOutput; ++i) { + int16_t functId = getExprFunctionId(&pExprs[i]); + + if (functId == FUNCTION_TOP || functId == FUNCTION_BOTTOM) { + int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); + if (j < 0 || j >= pTableInfo->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } else { + SColumnInfo* pCol = &pTableInfo->colList[j]; +// int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i, +// &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL); +// assert(ret == TSDB_CODE_SUCCESS); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +// TODO tag length should be passed from client, refactor +int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo) { + *pExprInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + code = initUdfInfo(pUdfInfo); + if (code) { + return code; + } + + SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); + if (pExprs == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + bool isSuperTable = /*QUERY_IS_STABLE_QUERY(queryType);*/ true; + int16_t tagLen = 0; + + for (int32_t i = 0; i < numOfOutput; ++i) { + pExprs[i].base = *pExprMsg[i]; + + memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param)); + for (int32_t j = 0; j < pExprMsg[i]->numOfParams; ++j) { + taosVariantAssign(&pExprs[i].base.param[j], &pExprMsg[i]->param[j]); + } + + int16_t type = 0; + int16_t bytes = 0; + + // parse the arithmetic expression + int32_t functionId = getExprFunctionId(&pExprs[i]); + if (functionId == FUNCTION_ARITHM) { + code = buildArithmeticExprFromMsg(&pExprs[i], pMsg); + + if (code != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return code; + } + + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypes[type].bytes; + } else if (functionId == FUNCTION_BLKINFO) { + SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN}; + type = s.type; + bytes = s.bytes; + } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && functionId == FUNCTION_TAGPRJ) { // parse the normal column + SSchema* s = tGetTbnameColumnSchema(); + type = s->type; + bytes = s->bytes; + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) { + // it is a user-defined constant value column + assert(functionId == FUNCTION_PRJ); + + type = pExprs[i].base.param[1].nType; + bytes = pExprs[i].base.param[1].nLen; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { + int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); + if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + } else { + if (j < PRIMARYKEY_TIMESTAMP_COL_ID || j >= pTableInfo->numOfCols) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + } + + if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { + SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pTableInfo->colList[j]; + type = pCol->type; + bytes = pCol->bytes; + } else { + SSchema* s = tGetTbnameColumnSchema(); + + type = s->type; + bytes = s->bytes; + } + +// if (pExprs[i].base.flist.numOfFilters > 0) { +// int32_t ret = cloneExprFilterInfo(&pExprs[i].base.flist.filterInfo, pExprMsg[i]->flist.filterInfo, +// pExprMsg[i]->flist.numOfFilters); +// if (ret) { +// tfree(pExprs); +// return ret; +// } +// } + } + + int32_t param = (int32_t)pExprs[i].base.param[0].i; +// if (functionId != FUNCTION_ARITHM && +// (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { +// tfree(pExprs); +// return TSDB_CODE_QRY_INVALID_MSG; +// } + + // todo remove it + SResultDataInfo info; + if (getResultDataInfo(type, bytes, functionId, param, &info, 0, isSuperTable/*, pUdfInfo*/) != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + + if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { + tagLen += pExprs[i].base.resSchema.bytes; + } + + assert(isValidDataType(pExprs[i].base.resSchema.type)); + } + + // the tag length is affected by other tag columns, so this should be update. + updateOutputBufForTopBotQuery(pTableInfo, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable); + + *pExprInfo = pExprs; + return TSDB_CODE_SUCCESS; +} + +int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { + tExprNode* expr = NULL; + + TRY(TSDB_MAX_TAG_CONDITIONS) { + expr = exprTreeFromBinary(data, len); + } CATCH( code ) { + CLEANUP_EXECUTE(); + return code; + } END_TRY + + if (expr == NULL) { + //qError("failed to create expr tree"); + return TSDB_CODE_QRY_APP_ERROR; + } + +// int32_t ret = filterInitFromTree(expr, pFilters, 0); +// tExprTreeDestroy(expr, NULL); + +// return ret; +} + + +// todo refactor +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo *pUdfInfo) { +// *pExprInfo = NULL; +// int32_t code = TSDB_CODE_SUCCESS; +// +// SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); +// if (pExprs == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } +// +// bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); +// +// for (int32_t i = 0; i < numOfOutput; ++i) { +// pExprs[i].base = *pExpr[i]; +// memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param)); +// +// for (int32_t j = 0; j < pExpr[i]->numOfParams; ++j) { +// taosVariantAssign(&pExprs[i].base.param[j], &pExpr[i]->param[j]); +// } +// +// pExprs[i].base.resSchema.type = 0; +// +// int16_t type = 0; +// int16_t bytes = 0; +// +// // parse the arithmetic expression +// if (pExprs[i].base.functionId == FUNCTION_ARITHM) { +// code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); +// +// if (code != TSDB_CODE_SUCCESS) { +// tfree(pExprs); +// return code; +// } +// +// type = TSDB_DATA_TYPE_DOUBLE; +// bytes = tDataTypes[type].bytes; +// } else { +// int32_t index = pExprs[i].base.colInfo.colIndex; +// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.colInfo.colId); +// +// type = prevExpr[index].base.resSchema.type; +// bytes = prevExpr[index].base.resSchema.bytes; +// } +// +// int32_t param = (int32_t)pExprs[i].base.param[0].i; +// if (getResultDataInfo(type, bytes, functionId, param, &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, +// &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { +// tfree(pExprs); +// return TSDB_CODE_QRY_INVALID_MSG; +// } +// +// assert(isValidDataType(pExprs[i].base.resSchema.type)); +// } +// +// *pExprInfo = pExprs; + return TSDB_CODE_SUCCESS; +} + +SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { + if (pQueryMsg->numOfGroupCols == 0) { + return NULL; + } + + // using group by tag columns + SGroupbyExpr *pGroupbyExpr = (SGroupbyExpr *)calloc(1, sizeof(SGroupbyExpr)); + if (pGroupbyExpr == NULL) { + *code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pGroupbyExpr->orderType = pQueryMsg->orderType; + pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; + + pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); + for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { + taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); + } + + return pGroupbyExpr; +} + +//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId) { +// *pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * numOfFilterCols); +// if (*pFilterInfo == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } +// +// for (int32_t i = 0, j = 0; i < numOfCols; ++i) { +// if (pCols[i].flist.numOfFilters > 0) { +// SSingleColumnFilterInfo* pFilter = &((*pFilterInfo)[j]); +// +// memcpy(&pFilter->info, &pCols[i], sizeof(SColumnInfo)); +// pFilter->info = pCols[i]; +// +// pFilter->numOfFilters = pCols[i].flist.numOfFilters; +// pFilter->pFilters = calloc(pFilter->numOfFilters, sizeof(SColumnFilterElem)); +// if (pFilter->pFilters == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } +// +// for (int32_t f = 0; f < pFilter->numOfFilters; ++f) { +// SColumnFilterElem* pSingleColFilter = &pFilter->pFilters[f]; +// pSingleColFilter->filterInfo = pCols[i].flist.filterInfo[f]; +// +// int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; +// int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; +// if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { +// //qError("QInfo:0x%"PRIx64" invalid filter info", qId); +// return TSDB_CODE_QRY_INVALID_MSG; +// } +// +// pSingleColFilter->fp = getFilterOperator(lower, upper); +// if (pSingleColFilter->fp == NULL) { +// //qError("QInfo:0x%"PRIx64" invalid filter info", qId); +// return TSDB_CODE_QRY_INVALID_MSG; +// } +// +// pSingleColFilter->bytes = pCols[i].bytes; +// +// if (lower == TSDB_RELATION_IN) { +//// buildFilterSetFromBinary(&pSingleColFilter->q, (char *)(pSingleColFilter->filterInfo.pz), (int32_t)(pSingleColFilter->filterInfo.len)); +// } +// } +// +// j++; +// } +// } +// +// return TSDB_CODE_SUCCESS; +//} + +void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols) { +// for (int32_t i = 0; i < numOfFilterCols; ++i) { +// if (pFilterInfo[i].numOfFilters > 0) { +// if (pFilterInfo[i].pFilters->filterInfo.lowerRelOptr == TSDB_RELATION_IN) { +// taosHashCleanup((SHashObj *)(pFilterInfo[i].pFilters->q)); +// } +// tfree(pFilterInfo[i].pFilters); +// } +// } +// +// tfree(pFilterInfo); + return NULL; +} + +int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { + for (int32_t i = 0; i < pQueryAttr->numOfCols; ++i) { +// if (pQueryAttr->tableCols[i].flist.numOfFilters > 0 && pQueryAttr->tableCols[i].flist.filterInfo != NULL) { +// pQueryAttr->numOfFilterCols++; +// } + } + + if (pQueryAttr->numOfFilterCols == 0) { + return TSDB_CODE_SUCCESS; + } + + doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, + &pQueryAttr->pFilterInfo, qId); + + pQueryAttr->createFilterOperator = true; + + return TSDB_CODE_SUCCESS; +} + +static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { + assert(pQueryAttr->pExpr1 != NULL && pQueryAttr != NULL); + + for (int32_t k = 0; k < pQueryAttr->numOfOutput; ++k) { + SSqlExpr *pSqlExprMsg = &pQueryAttr->pExpr1[k].base; +// if (pSqlExprMsg->functionId == FUNCTION_ARITHM) { +// continue; +// } + + // todo opt performance + SColIndex *pColIndex = &pSqlExprMsg->colInfo; + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + int32_t f = 0; + for (f = 0; f < pQueryAttr->numOfCols; ++f) { + if (pColIndex->colId == pQueryAttr->tableCols[f].colId) { + pColIndex->colIndex = f; + break; + } + } + + assert(f < pQueryAttr->numOfCols); + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns + } else { + int32_t f = 0; + for (f = 0; f < pQueryAttr->numOfTags; ++f) { + if (pColIndex->colId == pQueryAttr->tagColList[f].colId) { + pColIndex->colIndex = f; + break; + } + } + + assert(f < pQueryAttr->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); + } + } +} + +void setResultBufSize(SQueryAttr* pQueryAttr, SRspResultInfo* pResultInfo) { + const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); + + // the minimum number of rows for projection query + const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; + const int32_t DEFAULT_MIN_ROWS = 4096; + + const float THRESHOLD_RATIO = 0.85f; + + if (isProjQuery(pQueryAttr)) { + int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; + if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { + numOfRes = MIN_ROWS_FOR_PRJ_QUERY; + } + + pResultInfo->capacity = numOfRes; + } else { // in case of non-prj query, a smaller output buffer will be used. + pResultInfo->capacity = DEFAULT_MIN_ROWS; + } + + pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); + pResultInfo->total = 0; +} + +FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { + return ((SQInfo *)qHandle)->qId == qId; +} + +SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, + SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, + char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo) { + int16_t numOfCols = pQueryMsg->numOfCols; + int16_t numOfOutput = pQueryMsg->numOfOutput; + + SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); + if (pQInfo == NULL) { + goto _cleanup_qinfo; + } + + pQInfo->qId = qId; + pQInfo->startExecTs = 0; + + pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; + + // to make sure third party won't overwrite this structure + pQInfo->signature = pQInfo; + SQueryAttr* pQueryAttr = &pQInfo->query; + pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; + + pQueryAttr->tableGroupInfo = *pTableGroupInfo; + pQueryAttr->numOfCols = numOfCols; + pQueryAttr->numOfOutput = numOfOutput; + pQueryAttr->limit.limit = pQueryMsg->limit; + pQueryAttr->limit.offset = pQueryMsg->offset; + pQueryAttr->order.order = pQueryMsg->order; + pQueryAttr->order.orderColId = pQueryMsg->orderColId; + pQueryAttr->pExpr1 = pExprs; + pQueryAttr->pExpr2 = pSecExprs; + pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; + pQueryAttr->pGroupbyExpr = pGroupbyExpr; + memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval)); + pQueryAttr->fillType = pQueryMsg->fillType; + pQueryAttr->numOfTags = pQueryMsg->numOfTags; + pQueryAttr->tagColList = pTagCols; + pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; + pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; + pQueryAttr->sw = pQueryMsg->sw; + pQueryAttr->vgId = vgId; + + pQueryAttr->stableQuery = pQueryMsg->stableQuery; + pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; + pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; + pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; + pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo; + pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; + pQueryAttr->stabledev = pQueryMsg->stabledev; + pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; + pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; + pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; + pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; + pQueryAttr->stateWindow = pQueryMsg->stateWindow; + pQueryAttr->vgId = vgId; +// pQueryAttr->pFilters = pFilters; + + pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); + if (pQueryAttr->tableCols == NULL) { + goto _cleanup; + } + + pQueryAttr->srcRowSize = 0; + pQueryAttr->maxTableColumnWidth = 0; + for (int16_t i = 0; i < numOfCols; ++i) { + pQueryAttr->tableCols[i] = pQueryMsg->tableCols[i]; +// pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pQueryMsg->tableCols[i].flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); + + pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; + if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { + pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; + } + } + + for (int16_t col = 0; col < numOfOutput; ++col) { + assert(pExprs[col].base.resSchema.bytes > 0); + pQueryAttr->resultRowSize += pExprs[col].base.resSchema.bytes; + + // keep the tag length + if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { + pQueryAttr->tagLen += pExprs[col].base.resSchema.bytes; + } + +// if (pExprs[col].base.flist.filterInfo) { +// ++pQueryAttr->havingNum; +// } + } + + doUpdateExprColumnIndex(pQueryAttr); + + if (pSecExprs != NULL) { + int32_t resultRowSize = 0; + + // calculate the result row size + for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) { + assert(pSecExprs[col].base.resSchema.bytes > 0); + resultRowSize += pSecExprs[col].base.resSchema.bytes; + } + + if (resultRowSize > pQueryAttr->resultRowSize) { + pQueryAttr->resultRowSize = resultRowSize; + } + } + + if (pQueryAttr->fillType != TSDB_FILL_NONE) { + pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput); + if (pQueryAttr->fillVal == NULL) { + goto _cleanup; + } + + // the first column is the timestamp + memcpy(pQueryAttr->fillVal, (char *)pQueryMsg->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t)); + } + + size_t numOfGroups = 0; + if (pTableGroupInfo->pGroupList != NULL) { + numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); + STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; + + pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); + pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; + pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + } + + pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pQInfo->rspContext = NULL; + pQInfo->sql = sql; + pthread_mutex_init(&pQInfo->lock, NULL); + tsem_init(&pQInfo->ready, 0, 0); + + pQueryAttr->window = pQueryMsg->window; + updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + STimeWindow window = pQueryAttr->window; + + int32_t index = 0; + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); + + size_t s = taosArrayGetSize(pa); + SArray* p1 = taosArrayInit(s, POINTER_BYTES); + if (p1 == NULL) { + goto _cleanup; + } + + taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); + + for(int32_t j = 0; j < s; ++j) { +// STableKeyInfo* info = taosArrayGet(pa, j); +// window.skey = info->lastKey; +// +// void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); +// STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); +// if (item == NULL) { +// goto _cleanup; +// } +// +// item->groupIndex = i; +// taosArrayPush(p1, &item); + +// STableId* id = TSDB_TABLEID(info->pTable); +// taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); +// index += 1; + } + } + + colIdCheck(pQueryAttr, pQInfo->qId); + +// int32_t functionId = getExprFunctionId(&pExpr[0]); +// pQInfo->query.queryBlockDist = (functionId == FUNCTION_BLKINFO); + + //qDebug("qmsg:%p vgId:%d, QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->query.vgId, pQInfo->qId, pQInfo); + return pQInfo; + +_cleanup_qinfo: +// tsdbDestroyTableGroup(pTableGroupInfo); + + if (pGroupbyExpr != NULL) { + taosArrayDestroy(pGroupbyExpr->columnInfo); + free(pGroupbyExpr); + } + + tfree(pTagCols); + for (int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = &pExprs[i]; + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(pExprInfo->pExpr, NULL); + pExprInfo->pExpr = NULL; + } + +// if (pExprInfo->base.flist.filterInfo) { +// freeColumnFilterInfo(pExprInfo->base.flist.filterInfo, pExprInfo->base.flist.numOfFilters); +// } + } + + tfree(pExprs); + +// filterFreeInfo(pFilters); + +_cleanup: + freeQInfo(pQInfo); + return NULL; +} + +bool isValidQInfo(void *param) { + SQInfo *pQInfo = (SQInfo *)param; + if (pQInfo == NULL) { + return false; + } + + /* + * pQInfo->signature may be changed by another thread, so we assign value of signature + * into local variable, then compare by using local variable + */ + uint64_t sig = (uint64_t)pQInfo->signature; + return (sig == (uint64_t)pQInfo); +} + +int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, + int32_t prevResultLen, void* merger) { + int32_t code = TSDB_CODE_SUCCESS; + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + pRuntimeEnv->qinfo = pQInfo; + + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + STSBuf *pTsBuf = NULL; + + if (pTsBufInfo->tsLen > 0) { // open new file to save the result + char* tsBlock = start + pTsBufInfo->tsOffset; + pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pTsBufInfo->tsNumOfBlocks, pTsBufInfo->tsLen, pTsBufInfo->tsOrder, + pQueryAttr->vgId); + + if (pTsBuf == NULL) { + code = TSDB_CODE_QRY_NO_DISKSPACE; + goto _error; + } + tsBufResetPos(pTsBuf); + bool ret = tsBufNextPos(pTsBuf); + UNUSED(ret); + } + + SArray* prevResult = NULL; + if (prevResultLen > 0) { + prevResult = interResFromBinary(param->prevResult, prevResultLen); + pRuntimeEnv->prevResult = prevResult; + } + + pRuntimeEnv->currentOffset = pQueryAttr->limit.offset; + if (tsdb != NULL) { +// pQueryAttr->precision = tsdbGetCfg(tsdb)->precision; + } + + if ((QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.skey > pQueryAttr->window.ekey)) || + (!QUERY_IS_ASC_QUERY(pQueryAttr) && (pQueryAttr->window.ekey > pQueryAttr->window.skey))) { + //qDebug("QInfo:0x%"PRIx64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQueryAttr->window.skey, +// pQueryAttr->window.ekey, pQueryAttr->order.order); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; + // todo free memory + return TSDB_CODE_SUCCESS; + } + + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { + //qDebug("QInfo:0x%"PRIx64" no table qualified for tag filter, abort query", pQInfo->qId); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + } + + // filter the qualified + if ((code = doInitQInfo(pQInfo, pTsBuf, tsdb, sourceOptr, param->tableScanOperator, param->pOperator, merger)) != TSDB_CODE_SUCCESS) { + goto _error; + } + + return code; + +_error: + // table query ref will be decrease during error handling + freeQInfo(pQInfo); + return code; +} + +//TODO refactor +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { + if (pFilter == NULL || numOfFilters == 0) { + return; + } + + for (int32_t i = 0; i < numOfFilters; i++) { + if (pFilter[i].filterstr && pFilter[i].pz) { + free((void*)(pFilter[i].pz)); + } + } + + free(pFilter); +} + +static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo) { + if (pTableqinfoGroupInfo->pGroupList != NULL) { + int32_t numOfGroups = (int32_t) taosArrayGetSize(pTableqinfoGroupInfo->pGroupList); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = taosArrayGetP(pTableqinfoGroupInfo->pGroupList, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(p, j); + destroyTableQueryInfoImpl(item); + } + + taosArrayDestroy(p); + } + } + + taosArrayDestroy(pTableqinfoGroupInfo->pGroupList); + taosHashCleanup(pTableqinfoGroupInfo->map); + + pTableqinfoGroupInfo->pGroupList = NULL; + pTableqinfoGroupInfo->map = NULL; + pTableqinfoGroupInfo->numOfTables = 0; +} + +void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { + if (pExprInfo == NULL) { + assert(numOfExpr == 0); + return NULL; + } + + for (int32_t i = 0; i < numOfExpr; ++i) { + if (pExprInfo[i].pExpr != NULL) { + tExprTreeDestroy(pExprInfo[i].pExpr, NULL); + } + +// if (pExprInfo[i].base.flist.filterInfo) { +// freeColumnFilterInfo(pExprInfo[i].base.flist.filterInfo, pExprInfo[i].base.flist.numOfFilters); +// } + + for(int32_t j = 0; j < pExprInfo[i].base.numOfParams; ++j) { + taosVariantDestroy(&pExprInfo[i].base.param[j]); + } + } + + tfree(pExprInfo); + return NULL; +} + +void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { + if (pColumnInfo != NULL) { + assert(numOfCols >= 0); + + for (int32_t i = 0; i < numOfCols; i++) { + freeColumnFilterInfo(pColumnInfo[i].flist.filterInfo, pColumnInfo[i].flist.numOfFilters); + } + + tfree(pColumnInfo); + } + + return NULL; +} + +void freeQInfo(SQInfo *pQInfo) { + if (!isValidQInfo(pQInfo)) { + return; + } + + //qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); + + doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); + + SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + freeQueryAttr(pQueryAttr); + +// tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); + + tfree(pQInfo->pBuf); + tfree(pQInfo->sql); + + taosArrayDestroy(pQInfo->summary.queryProfEvents); + taosHashCleanup(pQInfo->summary.operatorProfResults); + + taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); + pQInfo->signature = 0; + + //qDebug("QInfo:0x%"PRIx64" QInfo is freed", pQInfo->qId); + + tfree(pQInfo); +} + +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { + // the remained number of retrieved rows, not the interpolated result + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; + + // load data from file to msg buffer + if (pQueryAttr->tsCompQuery) { + SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); + FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor + + // make sure file exist + if (f) { + off_t s = lseek(fileno(f), 0, SEEK_END); + assert(s == pRuntimeEnv->outputBuf->info.rows); + + //qDebug("QInfo:0x%"PRIx64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s); + if (fseek(f, 0, SEEK_SET) >= 0) { + size_t sz = fread(data, 1, s, f); + if(sz < s) { // todo handle error + //qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); + assert(0); + } + } else { + UNUSED(s); + //qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); + assert(0); + } + + // dump error info + if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { +// qDump(data, s); + assert(0); + } + + fclose(f); + *(FILE **)pColInfoData->pData = NULL; + } + + // all data returned, set query over + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } + } else { + doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); + } + + //qDebug("QInfo:0x%"PRIx64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId, +// pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); + + if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { + //qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } + + return TSDB_CODE_SUCCESS; +} + +bool doBuildResCheck(SQInfo* pQInfo) { + bool buildRes = false; + + pthread_mutex_lock(&pQInfo->lock); + + pQInfo->dataReady = QUERY_RESULT_READY; + buildRes = needBuildResAfterQueryComplete(pQInfo); + + // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is + // put into task to be executed. + assert(pQInfo->owner == taosGetSelfPthreadId()); + pQInfo->owner = 0; + + pthread_mutex_unlock(&pQInfo->lock); + + // used in retrieve blocking model. + tsem_post(&pQInfo->ready); + return buildRes; +} + +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { + if (val == NULL) { + setNull(output, type, bytes); + return; + } + + if (IS_VAR_DATA_TYPE(type)) { + // Binary data overflows for sort of unknown reasons. Let trim the overflow data + if (varDataTLen(val) > bytes) { + int32_t maxLen = bytes - VARSTR_HEADER_SIZE; + int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); + memcpy(varDataVal(output), varDataVal(val), len); + varDataSetLen(output, len); + } else { + varDataCopy(output, val); + } + } else { + memcpy(output, val, bytes); + } +} + +static int64_t getQuerySupportBufSize(size_t numOfTables) { + size_t s1 = sizeof(STableQueryInfo); + size_t s2 = sizeof(SHashNode); + +// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb + return (int64_t)((s1 + s2) * 1.5 * numOfTables); +} + +int32_t checkForQueryBuf(size_t numOfTables) { + int64_t t = getQuerySupportBufSize(numOfTables); + if (tsQueryBufferSizeBytes < 0) { + return TSDB_CODE_SUCCESS; + } else if (tsQueryBufferSizeBytes > 0) { + + while(1) { + int64_t s = tsQueryBufferSizeBytes; + int64_t remain = s - t; + if (remain >= 0) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { + return TSDB_CODE_SUCCESS; + } + } else { + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; + } + } + } + + // disable query processing if the value of tsQueryBufferSize is zero. + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; +} + +bool checkNeedToCompressQueryCol(SQInfo *pQInfo) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; + + SSDataBlock* pRes = pRuntimeEnv->outputBuf; + + if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) <= 0) { + return false; + } + + int32_t numOfRows = pQueryAttr->pExpr2 ? GET_NUM_OF_RESULTS(pRuntimeEnv) : pRes->info.rows; + int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; + + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + int32_t colSize = pColRes->info.bytes * numOfRows; + if (NEEDTO_COMPRESS_QUERY(colSize)) { + return true; + } + } + + return false; +} + +void releaseQueryBuf(size_t numOfTables) { + if (tsQueryBufferSizeBytes < 0) { + return; + } + + int64_t t = getQuerySupportBufSize(numOfTables); + + // restore value is not enough buffer available + atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); +} + +void freeQueryAttr(SQueryAttr* pQueryAttr) { + if (pQueryAttr != NULL) { + if (pQueryAttr->fillVal != NULL) { + tfree(pQueryAttr->fillVal); + } + + pQueryAttr->pFilterInfo = doDestroyFilterInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols); + + pQueryAttr->pExpr1 = destroyQueryFuncExpr(pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + pQueryAttr->pExpr2 = destroyQueryFuncExpr(pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); + pQueryAttr->pExpr3 = destroyQueryFuncExpr(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); + + tfree(pQueryAttr->tagColList); + tfree(pQueryAttr->pFilterInfo); + + pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); + + if (pQueryAttr->pGroupbyExpr != NULL) { + taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); + tfree(pQueryAttr->pGroupbyExpr); + } + +// filterFreeInfo(pQueryAttr->pFilters); + } +} + diff --git a/source/libs/executor/src/tfilter.c b/source/libs/executor/src/tfilter.c new file mode 100644 index 0000000000..48662f3443 --- /dev/null +++ b/source/libs/executor/src/tfilter.c @@ -0,0 +1,3521 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" +#include +#include "thash.h" +//#include "queryLog.h" +#include "tcompare.h" +#include "tfilter.h" + +OptrStr gOptrStr[] = { + {TSDB_RELATION_INVALID, "invalid"}, + {TSDB_RELATION_LESS, "<"}, + {TSDB_RELATION_GREATER, ">"}, + {TSDB_RELATION_EQUAL, "="}, + {TSDB_RELATION_LESS_EQUAL, "<="}, + {TSDB_RELATION_GREATER_EQUAL, ">="}, + {TSDB_RELATION_NOT_EQUAL, "!="}, + {TSDB_RELATION_LIKE, "like"}, + {TSDB_RELATION_MATCH, "match"}, + {TSDB_RELATION_MATCH, "nmatch"}, + {TSDB_RELATION_ISNULL, "is null"}, + {TSDB_RELATION_NOTNULL, "not null"}, + {TSDB_RELATION_IN, "in"}, + {TSDB_RELATION_AND, "and"}, + {TSDB_RELATION_OR, "or"}, + {TSDB_RELATION_NOT, "not"} +}; + +static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) { + const SSchema *sch1 = desc1; + const SSchema *sch2 = desc2; + + return sch1->colId != sch2->colId; +} + +static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) { + const SVariant *val1 = desc1; + const SVariant *val2 = desc2; + + return taosVariantCompare(val1, val2); +} + + +filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = { + NULL, + filterFieldColDescCompare, + filterFieldValDescCompare +}; + +bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0; +} +bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0; +} +bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(minv, maxr) <= 0; +} +bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(minv, maxr) < 0; +} +bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0; +} +bool filterRangeCompee (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) < 0; +} +bool filterRangeCompei (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) > 0 && cfunc(minv, maxr) <= 0; +} +bool filterRangeCompie (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { + return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) < 0; +} + +rangeCompFunc filterGetRangeCompFunc(char sflag, char eflag) { + if (FILTER_GET_FLAG(sflag, RANGE_FLG_NULL)) { + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompLe; + } + + return filterRangeCompLi; + } + + if (FILTER_GET_FLAG(eflag, RANGE_FLG_NULL)) { + if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompGe; + } + + return filterRangeCompGi; + } + + if (FILTER_GET_FLAG(sflag, RANGE_FLG_EXCLUDE)) { + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompee; + } + + return filterRangeCompei; + } + + if (FILTER_GET_FLAG(eflag, RANGE_FLG_EXCLUDE)) { + return filterRangeCompie; + } + + return filterRangeCompii; +} + +rangeCompFunc gRangeCompare[] = {filterRangeCompee, filterRangeCompei, filterRangeCompie, filterRangeCompii, filterRangeCompGe, + filterRangeCompGi, filterRangeCompLe, filterRangeCompLi}; + + +int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { + if (optr2) { + assert(optr2 == TSDB_RELATION_LESS || optr2 == TSDB_RELATION_LESS_EQUAL); + + if (optr == TSDB_RELATION_GREATER) { + if (optr2 == TSDB_RELATION_LESS) { + return 0; + } + + return 1; + } + + if (optr2 == TSDB_RELATION_LESS) { + return 2; + } + + return 3; + } else { + switch (optr) { + case TSDB_RELATION_GREATER: + return 4; + case TSDB_RELATION_GREATER_EQUAL: + return 5; + case TSDB_RELATION_LESS: + return 6; + case TSDB_RELATION_LESS_EQUAL: + return 7; + default: + break; + } + } + + return -1; +} + +__compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal, + compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, + compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, + setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch +}; + +int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { + int8_t comparFn = 0; + + if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + return 15; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + return 16; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_FLOAT: + return 17; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + return 18; + default: + assert(0); + } + } + + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = 2; break; + case TSDB_DATA_TYPE_INT: comparFn = 0; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; + case TSDB_DATA_TYPE_BINARY: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = 19; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = 20; + } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + comparFn = 7; + } else if (optr == TSDB_RELATION_IN) { + comparFn = 8; + } else { /* normal relational comparFn */ + comparFn = 6; + } + + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + if (optr == TSDB_RELATION_MATCH) { + comparFn = 19; + } else if (optr == TSDB_RELATION_NMATCH) { + comparFn = 20; + } else if (optr == TSDB_RELATION_LIKE) { + comparFn = 9; + } else if (optr == TSDB_RELATION_IN) { + comparFn = 8; + } else { + comparFn = 10; + } + break; + } + + case TSDB_DATA_TYPE_UTINYINT: comparFn = 11; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = 12;break; + case TSDB_DATA_TYPE_UINT: comparFn = 13;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = 14;break; + + default: + comparFn = 0; + break; + } + + return comparFn; +} + + +static FORCE_INLINE int32_t filterCompareGroupCtx(const void *pLeft, const void *pRight) { + SFilterGroupCtx *left = *((SFilterGroupCtx**)pLeft), *right = *((SFilterGroupCtx**)pRight); + if (left->colNum > right->colNum) return 1; + if (left->colNum < right->colNum) return -1; + return 0; +} + +int32_t filterInitUnitsFields(SFilterInfo *info) { + info->unitSize = FILTER_DEFAULT_UNIT_SIZE; + info->units = calloc(info->unitSize, sizeof(SFilterUnit)); + + info->fields[FLD_TYPE_COLUMN].num = 0; + info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE; + info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE); + info->fields[FLD_TYPE_VALUE].num = 0; + info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE; + info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField)); + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilterRange* ra) { + SFilterRangeNode *r = NULL; + + if (ctx->rf) { + r = ctx->rf; + ctx->rf = ctx->rf->next; + r->prev = NULL; + r->next = NULL; + } else { + r = calloc(1, sizeof(SFilterRangeNode)); + } + + FILTER_COPY_RA(&r->ra, ra); + + return r; +} + +void* filterInitRangeCtx(int32_t type, int32_t options) { + if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + //qError("not supported range type:%d", type); + return NULL; + } + + SFilterRangeCtx *ctx = calloc(1, sizeof(SFilterRangeCtx)); + + ctx->type = type; + ctx->options = options; + ctx->pCompareFunc = getComparFunc(type, 0); + + return ctx; +} + + +int32_t filterResetRangeCtx(SFilterRangeCtx *ctx) { + ctx->status = 0; + + if (ctx->rf == NULL) { + ctx->rf = ctx->rs; + ctx->rs = NULL; + return TSDB_CODE_SUCCESS; + } + + ctx->isnull = false; + ctx->notnull = false; + ctx->isrange = false; + + SFilterRangeNode *r = ctx->rf; + + while (r && r->next) { + r = r->next; + } + + r->next = ctx->rs; + ctx->rs = NULL; + return TSDB_CODE_SUCCESS; +} + +int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) { + filterResetRangeCtx(ctx); + + ctx->type = type; + ctx->options = options; + ctx->pCompareFunc = getComparFunc(type, 0); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type)); + if (sr == 0) { + FILTER_SET_FLAG(ra->sflag, RANGE_FLG_NULL); + } + } + + if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + int32_t er = cur->pCompareFunc(&ra->e, getDataMax(cur->type)); + if (er == 0) { + FILTER_SET_FLAG(ra->eflag, RANGE_FLG_NULL); + } + } + + + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL) && FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + *notNull = true; + } else { + *notNull = false; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddRangeOptr(void* h, uint8_t raOptr, int32_t optr, bool *empty, bool *all) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (optr == TSDB_RELATION_AND) { + SET_AND_OPTR(ctx, raOptr); + if (CHK_AND_OPTR(ctx) || (raOptr == FILTER_DUMMY_EMPTY_OPTR)) { + FILTER_SET_FLAG(ctx->status, MR_ST_EMPTY); + *empty = true; + } + } else { + SET_OR_OPTR(ctx, raOptr); + if (CHK_OR_OPTR(ctx)) { + FILTER_SET_FLAG(ctx->status, MR_ST_ALL); + *all = true; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterAddRangeImpl(void* h, SFilterRange* ra, int32_t optr) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (ctx->rs == NULL) { + if ((FILTER_GET_FLAG(ctx->status, MR_ST_START) == 0) + || (FILTER_GET_FLAG(ctx->status, MR_ST_ALL) && (optr == TSDB_RELATION_AND)) + || ((!FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) && (optr == TSDB_RELATION_OR))) { + APPEND_RANGE(ctx, ctx->rs, ra); + FILTER_SET_FLAG(ctx->status, MR_ST_START); + } + + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + int32_t cr = 0; + + if (optr == TSDB_RELATION_AND) { + while (r != NULL) { + cr = ctx->pCompareFunc(&r->ra.s, &ra->e); + if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { + FREE_FROM_RANGE(ctx, r); + break; + } + + cr = ctx->pCompareFunc(&ra->s, &r->ra.e); + if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { + rn = r->next; + FREE_RANGE(ctx, r); + r = rn; + continue; + } + + cr = ctx->pCompareFunc(&ra->s, &r->ra.s); + if (FILTER_GREATER(cr, ra->sflag, r->ra.sflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); + cr == 0 ? (r->ra.sflag |= ra->sflag) : (r->ra.sflag = ra->sflag); + } + + cr = ctx->pCompareFunc(&r->ra.e, &ra->e); + if (FILTER_GREATER(cr, r->ra.eflag, ra->eflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); + cr == 0 ? (r->ra.eflag |= ra->eflag) : (r->ra.eflag = ra->eflag); + break; + } + + r = r->next; + } + + return TSDB_CODE_SUCCESS; + } + + + //TSDB_RELATION_OR + + bool smerged = false; + bool emerged = false; + + while (r != NULL) { + cr = ctx->pCompareFunc(&r->ra.s, &ra->e); + if (FILTER_GREATER(cr, r->ra.sflag, ra->eflag)) { + if (emerged == false) { + INSERT_RANGE(ctx, r, ra); + } + + break; + } + + if (smerged == false) { + cr = ctx->pCompareFunc(&ra->s, &r->ra.e); + if (FILTER_GREATER(cr, ra->sflag, r->ra.eflag)) { + if (r->next) { + r= r->next; + continue; + } + + APPEND_RANGE(ctx, r, ra); + break; + } + + cr = ctx->pCompareFunc(&r->ra.s, &ra->s); + if (FILTER_GREATER(cr, r->ra.sflag, ra->sflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.s, &ra->s); + cr == 0 ? (r->ra.sflag &= ra->sflag) : (r->ra.sflag = ra->sflag); + } + + smerged = true; + } + + if (emerged == false) { + cr = ctx->pCompareFunc(&ra->e, &r->ra.e); + if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { + SIMPLE_COPY_VALUES((char *)&r->ra.e, &ra->e); + if (cr == 0) { + r->ra.eflag &= ra->eflag; + break; + } + + r->ra.eflag = ra->eflag; + emerged = true; + r = r->next; + continue; + } + + break; + } + + cr = ctx->pCompareFunc(&ra->e, &r->ra.e); + if (FILTER_GREATER(cr, ra->eflag, r->ra.eflag)) { + rn = r->next; + FREE_RANGE(ctx, r); + r = rn; + + continue; + } else { + SIMPLE_COPY_VALUES(&r->prev->ra.e, (char *)&r->ra.e); + cr == 0 ? (r->prev->ra.eflag &= r->ra.eflag) : (r->prev->ra.eflag = r->ra.eflag); + FREE_RANGE(ctx, r); + + break; + } + } + + if (ctx->rs && ctx->rs->next == NULL) { + bool notnull; + filterConvertRange(ctx, &ctx->rs->ra, ¬null); + if (notnull) { + bool all = false; + FREE_FROM_RANGE(ctx, ctx->rs); + filterAddRangeOptr(h, TSDB_RELATION_NOTNULL, optr, NULL, &all); + if (all) { + FILTER_SET_FLAG(ctx->status, MR_ST_ALL); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddRange(void* h, SFilterRange* ra, int32_t optr) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type)); + //FILTER_CLR_FLAG(ra->sflag, RA_NULL); + } + + if (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + SIMPLE_COPY_VALUES(&ra->e, getDataMax(ctx->type)); + //FILTER_CLR_FLAG(ra->eflag, RA_NULL); + } + + return filterAddRangeImpl(h, ra, optr); +} + + +int32_t filterAddRangeCtx(void *dst, void *src, int32_t optr) { + SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; + SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; + + assert(optr == TSDB_RELATION_OR); + + if (sctx->rs == NULL) { + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = sctx->rs; + + while (r) { + filterAddRange(dctx, &r->ra, optr); + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterCopyRangeCtx(void *dst, void *src) { + SFilterRangeCtx *dctx = (SFilterRangeCtx *)dst; + SFilterRangeCtx *sctx = (SFilterRangeCtx *)src; + + dctx->status = sctx->status; + + dctx->isnull = sctx->isnull; + dctx->notnull = sctx->notnull; + dctx->isrange = sctx->isrange; + + SFilterRangeNode *r = sctx->rs; + SFilterRangeNode *dr = dctx->rs; + + while (r) { + APPEND_RANGE(dctx, dr, &r->ra); + if (dr == NULL) { + dr = dctx->rs; + } else { + dr = dr->next; + } + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterFinishRange(void* h) { + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + if (FILTER_GET_FLAG(ctx->status, MR_ST_FIN)) { + return TSDB_CODE_SUCCESS; + } + + if (FILTER_GET_FLAG(ctx->options, FI_OPTION_TIMESTAMP)) { + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + + while (r && r->next) { + int64_t tmp = 1; + operateVal(&tmp, &r->ra.e, &tmp, TSDB_BINARY_OP_ADD, ctx->type); + if (ctx->pCompareFunc(&tmp, &r->next->ra.s) == 0) { + rn = r->next; + SIMPLE_COPY_VALUES((char *)&r->next->ra.s, (char *)&r->ra.s); + FREE_RANGE(ctx, r); + r = rn; + + continue; + } + + r = r->next; + } + } + + FILTER_SET_FLAG(ctx->status, MR_ST_FIN); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGetRangeNum(void* h, int32_t* num) { + filterFinishRange(h); + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + + *num = 0; + + SFilterRangeNode *r = ctx->rs; + + while (r) { + ++(*num); + r = r->next; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterGetRangeRes(void* h, SFilterRange *ra) { + filterFinishRange(h); + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + uint32_t num = 0; + SFilterRangeNode* r = ctx->rs; + + while (r) { + FILTER_COPY_RA(ra, &r->ra); + + ++num; + r = r->next; + ++ra; + } + + if (num == 0) { + //qError("no range result"); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterSourceRangeFromCtx(SFilterRangeCtx *ctx, void *sctx, int32_t optr, bool *empty, bool *all) { + SFilterRangeCtx *src = (SFilterRangeCtx *)sctx; + + if (src->isnull){ + filterAddRangeOptr(ctx, TSDB_RELATION_ISNULL, optr, empty, all); + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + if (src->notnull) { + filterAddRangeOptr(ctx, TSDB_RELATION_NOTNULL, optr, empty, all); + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + if (src->isrange) { + filterAddRangeOptr(ctx, 0, optr, empty, all); + + if (!(optr == TSDB_RELATION_OR && ctx->notnull)) { + filterAddRangeCtx(ctx, src, optr); + } + + if (FILTER_GET_FLAG(ctx->status, MR_ST_ALL)) { + *all = true; + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterFreeRangeCtx(void* h) { + if (h == NULL) { + return TSDB_CODE_SUCCESS; + } + + SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; + SFilterRangeNode *r = ctx->rs; + SFilterRangeNode *rn = NULL; + + while (r) { + rn = r->next; + free(r); + r = rn; + } + + r = ctx->rf; + while (r) { + rn = r->next; + free(r); + r = rn; + } + + free(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterDetachCnfGroup(SFilterGroup *gp1, SFilterGroup *gp2, SArray* group) { + SFilterGroup gp = {0}; + + gp.unitNum = gp1->unitNum + gp2->unitNum; + gp.unitIdxs = calloc(gp.unitNum, sizeof(*gp.unitIdxs)); + memcpy(gp.unitIdxs, gp1->unitIdxs, gp1->unitNum * sizeof(*gp.unitIdxs)); + memcpy(gp.unitIdxs + gp1->unitNum, gp2->unitIdxs, gp2->unitNum * sizeof(*gp.unitIdxs)); + + gp.unitFlags = NULL; + + taosArrayPush(group, &gp); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) { + int32_t leftSize = (int32_t)taosArrayGetSize(left); + int32_t rightSize = (int32_t)taosArrayGetSize(right); + +// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); +// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group"); + + for (int32_t l = 0; l < leftSize; ++l) { + SFilterGroup *gp1 = taosArrayGet(left, l); + + for (int32_t r = 0; r < rightSize; ++r) { + SFilterGroup *gp2 = taosArrayGet(right, r); + + filterDetachCnfGroup(gp1, gp2, group); + } + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) { + for (uint16_t i = 0; i < fields->num; ++i) { + if (0 == gDescCompare[type](fields->fields[i].desc, v)) { + return i; + } + } + + return -1; +} + + +int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { + if (type == FLD_TYPE_VALUE) { + if (info->pctx.valHash == false) { + //qError("value hash is empty"); + return -1; + } + + void *hv = taosHashGet(info->pctx.valHash, v, dataLen); + if (hv) { + return *(int32_t *)hv; + } + } + + return -1; +} + + +int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) { + int32_t idx = -1; + uint16_t *num; + + num = &info->fields[type].num; + + if (*num > 0) { + if (type == FLD_TYPE_COLUMN) { + idx = filterGetFiledByDesc(&info->fields[type], type, desc); + } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + idx = filterGetFiledByData(info, type, *data, dataLen); + } + } + + if (idx < 0) { + idx = *num; + if (idx >= info->fields[type].size) { + info->fields[type].size += FILTER_DEFAULT_FIELD_SIZE; + info->fields[type].fields = realloc(info->fields[type].fields, info->fields[type].size * sizeof(SFilterField)); + } + + info->fields[type].fields[idx].flag = type; + info->fields[type].fields[idx].desc = desc; + info->fields[type].fields[idx].data = data ? *data : NULL; + + if (type == FLD_TYPE_COLUMN) { + FILTER_SET_FLAG(info->fields[type].fields[idx].flag, FLD_DATA_NO_FREE); + } + + ++(*num); + + if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (info->pctx.valHash == NULL) { + info->pctx.valHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_VALUE_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + } + + taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); + } + } else { + if (freeIfExists) { + tfree(desc); + } + + if (data && freeIfExists) { + tfree(*data); + } + } + + fid->type = type; + fid->idx = idx; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { + filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); + + FILTER_SET_FLAG(field->flag, FLD_DESC_NO_FREE); + FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) { +// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node"); +// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR); + + int32_t type; + void *v; + + if (node->nodeType == TEXPR_BINARYEXPR_NODE) { + type = FLD_TYPE_COLUMN; + v = node->pSchema; + node->pSchema = NULL; + } else { + type = FLD_TYPE_VALUE; + v = node->pVal; + node->pVal = NULL; + } + + filterAddField(info, v, NULL, type, fid, 0, true); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) { + if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + if (info->pctx.unitHash == NULL) { + info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); + } else { + int64_t v = 0; + FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); + void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); + if (hu) { + *uidx = *(uint16_t *)hu; + return TSDB_CODE_SUCCESS; + } + } + } + + if (info->unitNum >= info->unitSize) { + uint16_t psize = info->unitSize; + info->unitSize += FILTER_DEFAULT_UNIT_SIZE; + info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit)); + memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); + } + + SFilterUnit *u = &info->units[info->unitNum]; + + u->compare.optr = optr; + u->left = *left; + if (right) { + u->right = *right; + } + + if (u->right.type == FLD_TYPE_VALUE) { + SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); + assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); + } else { + assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR); + } + + SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u); + assert(FILTER_GET_FLAG(col->flag, FLD_TYPE_COLUMN)); + + info->units[info->unitNum].compare.type = FILTER_GET_COL_FIELD_TYPE(col); + + *uidx = info->unitNum; + + if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) { + int64_t v = 0; + FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); + taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); + } + + ++info->unitNum; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { + if (group->unitNum >= group->unitSize) { + group->unitSize += FILTER_DEFAULT_UNIT_SIZE; + group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs)); + } + + group->unitIdxs[group->unitNum++] = unitIdx; + + return TSDB_CODE_SUCCESS; +} + +int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) { + SBufferReader br = tbufInitReader(buf, len, false); + uint32_t sType = tbufReadUint32(&br); + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false); + int32_t code = 0; + + int dummy = -1; + SVariant tmpVar = {0}; + size_t t = 0; + int32_t sz = tbufReadInt32(&br); + void *pvar = NULL; + int64_t val = 0; + int32_t bufLen = 0; + if (IS_NUMERIC_TYPE(sType)) { + bufLen = 60; // The maximum length of string that a number is converted to. + } else { + bufLen = 128; + } + + char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + + for (int32_t i = 0; i < sz; i++) { + switch (sType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + *(uint8_t *)&val = (uint8_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + *(uint16_t *)&val = (uint16_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + *(uint32_t *)&val = (uint32_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + *(uint64_t *)&val = (uint64_t)tbufReadInt64(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + *(double *)&val = tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(float *)&val = (float)tbufReadDouble(&br); + t = sizeof(val); + pvar = &val; + break; + } + case TSDB_DATA_TYPE_BINARY: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + pvar = (char *)tbufReadBinary(&br, &t); + break; + } + default: + taosHashCleanup(pObj); + *q = NULL; + assert(0); + } + + taosVariantCreateFromBinary(&tmpVar, (char *)pvar, t, sType); + + if (bufLen < t) { + tmp = realloc(tmp, t * TSDB_NCHAR_SIZE); + bufLen = (int32_t)t; + } + + bool converted = false; + char extInfo = 0; + + switch (tType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { +// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } +// +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { +// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } +// +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { +// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } +// +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (taosVariantDump(&tmpVar, (char *)&val, tType, false)) { + goto _return; + } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_FLOAT: { +// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) { +// if (converted) { +// taosVariantDestroy(&tmpVar); +// memset(&tmpVar, 0, sizeof(tmpVar)); +// continue; +// } + +// goto _return; +// } + pvar = &val; + t = sizeof(val); + break; + } + case TSDB_DATA_TYPE_BINARY: { + if (taosVariantDump(&tmpVar, tmp, tType, true)) { + goto _return; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + case TSDB_DATA_TYPE_NCHAR: { + if (taosVariantDump(&tmpVar, tmp, tType, true)) { + goto _return; + } + t = varDataLen(tmp); + pvar = varDataVal(tmp); + break; + } + default: + goto _return; + } + + taosHashPut(pObj, (char *)pvar, t, &dummy, sizeof(dummy)); + taosVariantDestroy(&tmpVar); + memset(&tmpVar, 0, sizeof(tmpVar)); + } + + *q = (void *)pObj; + pObj = NULL; + +_return: + taosVariantDestroy(&tmpVar); + taosHashCleanup(pObj); + tfree(tmp); + + return code; +} + + + +int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { + SFilterFieldId left = {0}, right = {0}; + + filterAddFieldFromNode(info, tree->_node.pLeft, &left); + + SVariant* var = tree->_node.pRight->pVal; + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + size_t len = 0; + uint16_t uidx = 0; + + if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { + void *data = NULL; + filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type); +// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + if (taosHashGetSize((SHashObj *)data) <= 0) { + filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + taosHashCleanup(data); + + return TSDB_CODE_SUCCESS; + } + + void *p = taosHashIterate((SHashObj *)data, NULL); + while(p) { + void* key = NULL; + len = 0; + + taosHashGetKey((SHashObj *)data, p, &key, &len); + void *fdata = NULL; + + if (IS_VAR_DATA_TYPE(type)) { + fdata = malloc(len + VARSTR_HEADER_SIZE); + varDataLen(fdata) = len; + memcpy(varDataVal(fdata), key, len); + len += VARSTR_HEADER_SIZE; + } else { + fdata = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(fdata, key); + len = tDataTypes[type].bytes; + } + + filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); + + filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + + p = taosHashIterate((SHashObj *)data, p); + } + + taosHashCleanup(data); + } else { + filterAddFieldFromNode(info, tree->_node.pRight, &right); + + filterAddUnit(info, tree->_node.optr, &left, &right, &uidx); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, uidx); + + taosArrayPush(group, &fgroup); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) { + SFilterFieldId left, right, *pright = &right; + int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint16_t flag = FLD_DESC_NO_FREE; + + filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); + SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); + FILTER_SET_FLAG(t->flag, flag); + + if (u->right.type == FLD_TYPE_VALUE) { + void *data = FILTER_UNIT_VAL_DATA(src, u); + if (IS_VAR_DATA_TYPE(type)) { + if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false); + + t = FILTER_GET_FIELD(dst, right); + + FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); + } else { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); + } + } else { + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); + } + + flag = FLD_DATA_NO_FREE; + t = FILTER_UNIT_RIGHT_FIELD(src, u); + FILTER_SET_FLAG(t->flag, flag); + } else { + pright = NULL; + } + + return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx); +} + +int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) { + SFilterUnit *u = &info->units[uidx]; + + u->compare.optr2 = optr; + u->right2 = *right; + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) { + SFilterFieldId left, right, right2; + uint16_t uidx = 0; + + SFilterField *col = FILTER_GET_COL_FIELD(src, cidx); + + filterAddColFieldFromField(dst, col, &left); + + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); + + if (optr == TSDB_RELATION_AND) { + if (ctx->isnull) { + assert(ctx->notnull == false && ctx->isrange == false); + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + + if (ctx->notnull) { + assert(ctx->isnull == false && ctx->isrange == false); + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + + if (!ctx->isrange) { + assert(ctx->isnull || ctx->notnull); + return TSDB_CODE_SUCCESS; + } + + assert(ctx->rs && ctx->rs->next == NULL); + + SFilterRange *ra = &ctx->rs->ra; + + assert(!((FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)))); + + if ((!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) && (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL))) { + __compar_fn_t func = getComparFunc(type, 0); + if (func(&ra->s, &ra->e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } else { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + void *data2 = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data2, &ra->e); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnitToGroup(g, uidx); + return TSDB_CODE_SUCCESS; + } + } + + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &ra->e); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + return TSDB_CODE_SUCCESS; + } + + // OR PROCESS + + SFilterGroup ng = {0}; + g = &ng; + + assert(ctx->isnull || ctx->notnull || ctx->isrange); + + if (ctx->isnull) { + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + taosArrayPush(res, g); + } + + if (ctx->notnull) { + assert(!ctx->isrange); + memset(g, 0, sizeof(*g)); + + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL, &uidx); + filterAddUnitToGroup(g, uidx); + taosArrayPush(res, g); + } + + if (!ctx->isrange) { + assert(ctx->isnull || ctx->notnull); + g->unitNum = 0; + return TSDB_CODE_SUCCESS; + } + + SFilterRangeNode *r = ctx->rs; + + while (r) { + memset(g, 0, sizeof(*g)); + + if ((!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) &&(!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL))) { + __compar_fn_t func = getComparFunc(type, 0); + if (func(&r->ra.s, &r->ra.e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } else { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + void *data2 = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data2, &r->ra.e); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &right2, uidx); + filterAddUnitToGroup(g, uidx); + } + + taosArrayPush(res, g); + + r = r->next; + + continue; + } + + if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.s); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &r->ra.e); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right, &uidx); + filterAddUnitToGroup(g, uidx); + } + + assert (g->unitNum > 0); + + taosArrayPush(res, g); + + r = r->next; + } + + g->unitNum = 0; + + return TSDB_CODE_SUCCESS; +} + + +static void filterFreeGroup(void *pItem) { + if (pItem == NULL) { + return; + } + + SFilterGroup* p = (SFilterGroup*) pItem; + tfree(p->unitIdxs); + tfree(p->unitFlags); +} + + +int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* leftGroup = NULL; + SArray* rightGroup = NULL; + + if (tree->nodeType != TEXPR_BINARYEXPR_NODE) { + //qError("invalid nodeType:%d", tree->nodeType); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (tree->_node.optr == TSDB_RELATION_AND) { + leftGroup = taosArrayInit(4, sizeof(SFilterGroup)); + rightGroup = taosArrayInit(4, sizeof(SFilterGroup)); + ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup)); + ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup)); + + ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup)); + + taosArrayDestroyEx(leftGroup, filterFreeGroup); + taosArrayDestroyEx(rightGroup, filterFreeGroup); + + return TSDB_CODE_SUCCESS; + } + + if (tree->_node.optr == TSDB_RELATION_OR) { + ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group)); + ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group)); + + return TSDB_CODE_SUCCESS; + } + + code = filterAddGroupUnitFromNode(info, tree, group); + + +_return: + + taosArrayDestroyEx(leftGroup, filterFreeGroup); + taosArrayDestroyEx(rightGroup, filterFreeGroup); + + return code; +} + +#if 0 +int32_t filterInitUnitFunc(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit* unit = &info->units[i]; + + info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + } + + return TSDB_CODE_SUCCESS; +} +#endif + +int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { +// tscError("invalid buf size"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *str = '"'; + memcpy(str + 1, buf, bufSize); + *(str + bufSize + 1) = '"'; + n = bufSize + 2; + break; + + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%d", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%d", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + + default: +// tscError("unsupported type:%d", type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *len = n; + + return TSDB_CODE_SUCCESS; +} + +void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) { + if (qDebugFlag & DEBUG_DEBUG) { +// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg); + + if (options == 0) { +// //qDebug("%s - FilterInfo:", msg); +// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema *sch = field->desc; +// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name); + } + + //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); + for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { + SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; + if (field->desc) { + SVariant *var = field->desc; + if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { + //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1)); + } else { + //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO + } + } else if (field->data) { + //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO + } + } + + //qDebug("UNIT Num:%u", info->unitNum); + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + int32_t type = FILTER_UNIT_DATA_TYPE(unit); + int32_t len = 0; + int32_t tlen = 0; + char str[512] = {0}; + + SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); + SSchema *sch = left->desc; + len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); + + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen); + } else { + strcat(str, "NULL"); + } + strcat(str, "]"); + + if (unit->compare.optr2) { + strcat(str, " && "); + sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str); + + if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { + SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen); + } else { + strcat(str, "NULL"); + } + strcat(str, "]"); + } + + //qDebug("%s", str); //TODO + } + + //qDebug("GROUP Num:%u", info->groupNum); + for (uint16_t i = 0; i < info->groupNum; ++i) { + SFilterGroup *group = &info->groups[i]; + //qDebug("Group%d : unit num[%u]", i, group->unitNum); + + for (uint16_t u = 0; u < group->unitNum; ++u) { + //qDebug("unit id:%u", group->unitIdxs[u]); + } + } + + return; + } + + if (options == 1) { + //qDebug("%s - RANGE info:", msg); + + //qDebug("RANGE Num:%u", info->colRangeNum); + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx *ctx = info->colRange[i]; + //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange); + if (ctx->isrange) { + SFilterRangeNode *r = ctx->rs; + while (r) { + char str[256] = {0}; + int32_t tlen = 0; + if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { + strcat(str,"(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + strcat(str, " - "); + if (FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { + strcat(str, "(NULL)"); + } else { + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"["); + converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen); + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]"); + } + //qDebug("range: %s", str); + + r = r->next; + } + } + } + + return; + } + + //qDebug("%s - Block Filter info:", msg); + + if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { + //qDebug("Flag:%s", "ALL"); + return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { + //qDebug("Flag:%s", "EMPTY"); + return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){ + //qDebug("Flag:%s", "ACTIVE"); + } + + //qDebug("GroupNum:%d", info->blkGroupNum); + uint16_t *unitIdx = info->blkUnits; + for (uint16_t i = 0; i < info->blkGroupNum; ++i) { + //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx); + uint16_t unitNum = *(unitIdx++); + for (uint16_t m = 0; m < unitNum; ++m) { + //qDebug("uidx[%d]", *(unitIdx++)); + } + } + } +} + +void filterFreeColInfo(void *data) { + SFilterColInfo* info = (SFilterColInfo *)data; + + if (info->info == NULL) { + return; + } + + if (info->type == RANGE_TYPE_VAR_HASH) { + //TODO + } else if (info->type == RANGE_TYPE_MR_CTX) { + filterFreeRangeCtx(info->info); + } else if (info->type == RANGE_TYPE_UNIT) { + taosArrayDestroy((SArray *)info->info); + } + + //NO NEED TO FREE UNIT + + info->type = 0; + info->info = NULL; +} + +void filterFreeColCtx(void *data) { + SFilterColCtx* ctx = (SFilterColCtx *)data; + + if (ctx->ctx) { + filterFreeRangeCtx(ctx->ctx); + } +} + + +void filterFreeGroupCtx(SFilterGroupCtx* gRes) { + if (gRes == NULL) { + return; + } + + tfree(gRes->colIdx); + + int16_t i = 0, j = 0; + + while (i < gRes->colNum) { + if (gRes->colInfo[j].info) { + filterFreeColInfo(&gRes->colInfo[j]); + ++i; + } + + ++j; + } + + tfree(gRes->colInfo); + tfree(gRes); +} + +void filterFreeField(SFilterField* field, int32_t type) { + if (field == NULL) { + return; + } + + if (!FILTER_GET_FLAG(field->flag, FLD_DESC_NO_FREE)) { + if (type == FLD_TYPE_VALUE) { + taosVariantDestroy(field->desc); + } + + tfree(field->desc); + } + + if (!FILTER_GET_FLAG(field->flag, FLD_DATA_NO_FREE)) { + if (FILTER_GET_FLAG(field->flag, FLD_DATA_IS_HASH)) { + taosHashCleanup(field->data); + } else { + tfree(field->data); + } + } +} + +void filterFreePCtx(SFilterPCtx *pctx) { + taosHashCleanup(pctx->valHash); + taosHashCleanup(pctx->unitHash); +} + +void filterFreeInfo(SFilterInfo *info) { + CHK_RETV(info == NULL); + + tfree(info->cunits); + tfree(info->blkUnitRes); + tfree(info->blkUnits); + + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { + for (uint16_t f = 0; f < info->fields[i].num; ++f) { + filterFreeField(&info->fields[i].fields[f], i); + } + + tfree(info->fields[i].fields); + } + + for (int32_t i = 0; i < info->groupNum; ++i) { + filterFreeGroup(&info->groups[i]); + } + + tfree(info->groups); + + tfree(info->units); + + tfree(info->unitRes); + + tfree(info->unitFlags); + + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + filterFreeRangeCtx(info->colRange[i]); + } + + tfree(info->colRange); + + filterFreePCtx(&info->pctx); + + if (!FILTER_GET_FLAG(info->status, FI_STATUS_CLONED)) { + tfree(info); + } +} + +int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) { + assert(extInfo > 0 || extInfo < 0); + + uint8_t optr = FILTER_UNIT_OPTR(unit); + switch (optr) { + case TSDB_RELATION_GREATER: + case TSDB_RELATION_GREATER_EQUAL: + unit->compare.optr = (extInfo > 0) ? FILTER_DUMMY_EMPTY_OPTR : TSDB_RELATION_NOTNULL; + break; + case TSDB_RELATION_LESS: + case TSDB_RELATION_LESS_EQUAL: + unit->compare.optr = (extInfo > 0) ? TSDB_RELATION_NOTNULL : FILTER_DUMMY_EMPTY_OPTR; + break; + case TSDB_RELATION_EQUAL: + unit->compare.optr = FILTER_DUMMY_EMPTY_OPTR; + break; + default: + assert(0); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterInitValFieldData(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit* unit = &info->units[i]; + if (unit->right.type != FLD_TYPE_VALUE) { + assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + continue; + } + + SFilterField* right = FILTER_UNIT_RIGHT_FIELD(info, unit); + + assert(FILTER_GET_FLAG(right->flag, FLD_TYPE_VALUE)); + + uint32_t type = FILTER_UNIT_DATA_TYPE(unit); + SFilterField* fi = right; + + SVariant* var = fi->desc; + + if (var == NULL) { + assert(fi->data != NULL); + continue; + } + + if (unit->compare.optr == TSDB_RELATION_IN) { + filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); +// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); + + continue; + } + + if (type == TSDB_DATA_TYPE_BINARY) { + size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; + fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + } else { + if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE + fi->data = calloc(var->nLen, tDataTypes[type].bytes); + for (int32_t a = 0; a < var->nLen; ++a) { + int64_t *v = taosArrayGet(var->arr, a); + assignVal((char *)fi->data + a * tDataTypes[type].bytes, (char *)v, 0, type); + } + + continue; + } else { + fi->data = calloc(1, sizeof(int64_t)); + } + } + + bool converted = false; + char extInfo = 0; +// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) { +// if (converted) { +// filterHandleValueExtInfo(unit, extInfo); +// +// continue; +// } +// //qError("dump value to type[%d] failed", type); +// return TSDB_CODE_TSC_INVALID_OPERATION; +// } + } + + return TSDB_CODE_SUCCESS; +} + + +bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) { + int32_t ret = func(left, right); + + switch (optr) { + case TSDB_RELATION_EQUAL: { + return ret == 0; + } + case TSDB_RELATION_NOT_EQUAL: { + return ret != 0; + } + case TSDB_RELATION_GREATER_EQUAL: { + return ret >= 0; + } + case TSDB_RELATION_GREATER: { + return ret > 0; + } + case TSDB_RELATION_LESS_EQUAL: { + return ret <= 0; + } + case TSDB_RELATION_LESS: { + return ret < 0; + } + case TSDB_RELATION_LIKE: { + return ret == 0; + } + case TSDB_RELATION_MATCH: { + return ret == 0; + } + case TSDB_RELATION_NMATCH: { + return ret == 0; + } + case TSDB_RELATION_IN: { + return ret == 1; + } + + default: + assert(false); + } + + return true; +} + + +int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRangeCtx *ctx, int32_t optr) { + int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint8_t uoptr = FILTER_UNIT_OPTR(u); + void *val = FILTER_UNIT_VAL_DATA(info, u); + SFilterRange ra = {0}; + int64_t tmp = 0; + + switch (uoptr) { + case TSDB_RELATION_GREATER: + SIMPLE_COPY_VALUES(&ra.s, val); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_EXCLUDE); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_GREATER_EQUAL: + SIMPLE_COPY_VALUES(&ra.s, val); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_LESS: + SIMPLE_COPY_VALUES(&ra.e, val); + FILTER_SET_FLAG(ra.eflag, RANGE_FLG_EXCLUDE); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_LESS_EQUAL: + SIMPLE_COPY_VALUES(&ra.e, val); + FILTER_SET_FLAG(ra.sflag, RANGE_FLG_NULL); + break; + case TSDB_RELATION_NOT_EQUAL: + assert(type == TSDB_DATA_TYPE_BOOL); + if (GET_INT8_VAL(val)) { + SIMPLE_COPY_VALUES(&ra.s, &tmp); + SIMPLE_COPY_VALUES(&ra.e, &tmp); + } else { + *(bool *)&tmp = true; + SIMPLE_COPY_VALUES(&ra.s, &tmp); + SIMPLE_COPY_VALUES(&ra.e, &tmp); + } + break; + case TSDB_RELATION_EQUAL: + SIMPLE_COPY_VALUES(&ra.s, val); + SIMPLE_COPY_VALUES(&ra.e, val); + break; + default: + assert(0); + } + + filterAddRange(ctx, &ra, optr); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterCompareRangeCtx(SFilterRangeCtx *ctx1, SFilterRangeCtx *ctx2, bool *equal) { + CHK_JMP(ctx1->status != ctx2->status); + CHK_JMP(ctx1->isnull != ctx2->isnull); + CHK_JMP(ctx1->notnull != ctx2->notnull); + CHK_JMP(ctx1->isrange != ctx2->isrange); + + SFilterRangeNode *r1 = ctx1->rs; + SFilterRangeNode *r2 = ctx2->rs; + + while (r1 && r2) { + CHK_JMP(r1->ra.sflag != r2->ra.sflag); + CHK_JMP(r1->ra.eflag != r2->ra.eflag); + CHK_JMP(r1->ra.s != r2->ra.s); + CHK_JMP(r1->ra.e != r2->ra.e); + + r1 = r1->next; + r2 = r2->next; + } + + CHK_JMP(r1 != r2); + + *equal = true; + + return TSDB_CODE_SUCCESS; + +_return: + *equal = false; + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) { + SArray* colArray = (SArray *)gRes->colInfo[colIdx].info; + int32_t size = (int32_t)taosArrayGetSize(colArray); + int32_t type = gRes->colInfo[colIdx].dataType; + SFilterRangeCtx* ctx = filterInitRangeCtx(type, 0); + + for (uint32_t i = 0; i < size; ++i) { + SFilterUnit* u = taosArrayGetP(colArray, i); + uint8_t optr = FILTER_UNIT_OPTR(u); + + filterAddRangeOptr(ctx, optr, TSDB_RELATION_AND, empty, NULL); + CHK_JMP(*empty); + + if (!FILTER_NO_MERGE_OPTR(optr)) { + filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND); + CHK_JMP(MR_EMPTY_RES(ctx)); + } + } + + taosArrayDestroy(colArray); + + FILTER_PUSH_CTX(gRes->colInfo[colIdx], ctx); + + return TSDB_CODE_SUCCESS; + +_return: + + *empty = true; + + filterFreeRangeCtx(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) { + bool empty = false; + uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t)); + uint16_t colIdxi = 0; + uint16_t gResIdx = 0; + + for (uint16_t i = 0; i < info->groupNum; ++i) { + SFilterGroup* g = info->groups + i; + + gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx)); + gRes[gResIdx]->colInfo = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(SFilterColInfo)); + colIdxi = 0; + empty = false; + + for (uint16_t j = 0; j < g->unitNum; ++j) { + SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j); + uint16_t cidx = FILTER_UNIT_COL_IDX(u); + + if (gRes[gResIdx]->colInfo[cidx].info == NULL) { + gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES); + colIdx[colIdxi++] = cidx; + ++gRes[gResIdx]->colNum; + } else { + if (!FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))) { + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + } + } + + FILTER_PUSH_UNIT(gRes[gResIdx]->colInfo[cidx], u); + } + + if (colIdxi > 1) { + qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0)); + } + + for (uint16_t l = 0; l < colIdxi; ++l) { + int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType; + + if (FILTER_NO_MERGE_DATA_TYPE(type)) { + continue; + } + + filterMergeUnits(info, gRes[gResIdx], colIdx[l], &empty); + + if (empty) { + break; + } + } + + if (empty) { + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + filterFreeGroupCtx(gRes[gResIdx]); + gRes[gResIdx] = NULL; + + continue; + } + + gRes[gResIdx]->colNum = colIdxi; + FILTER_COPY_IDX(&gRes[gResIdx]->colIdx, colIdx, colIdxi); + ++gResIdx; + } + + tfree(colIdx); + + *gResNum = gResIdx; + + if (gResIdx == 0) { + FILTER_SET_FLAG(info->status, FI_STATUS_EMPTY); + } + + return TSDB_CODE_SUCCESS; +} + +void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { + uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + bool equal = false; + + for (; m < gRes1->colNum; ++m) { + idx1 = gRes1->colIdx[m]; + + equal = false; + + for (; n < gRes2->colNum; ++n) { + idx2 = gRes2->colIdx[n]; + if (idx1 < idx2) { + *conflict = true; + return; + } + + if (idx1 > idx2) { + continue; + } + + if (FILTER_NO_MERGE_DATA_TYPE(gRes1->colInfo[idx1].dataType)) { + *conflict = true; + return; + } + + ++n; + equal = true; + break; + } + + if (!equal) { + *conflict = true; + return; + } + } + + *conflict = false; + return; +} + + +int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) { + SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx); + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + + if ((*ctx) == NULL) { + *ctx = filterInitRangeCtx(type, 0); + } else { + filterReuseRangeCtx(*ctx, type, 0); + } + + assert(gRes2->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + assert(gRes1->colInfo[cidx].type == RANGE_TYPE_MR_CTX); + + filterCopyRangeCtx(*ctx, gRes2->colInfo[cidx].info); + filterSourceRangeFromCtx(*ctx, gRes1->colInfo[cidx].info, optr, empty, all); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilterGroupCtx** gRes2, bool *all) { + bool conflict = false; + + filterCheckColConflict(*gRes1, *gRes2, &conflict); + if (conflict) { + return TSDB_CODE_SUCCESS; + } + + FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE); + + uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0; + bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum; + bool equal = false; + uint16_t equal1 = 0, equal2 = 0, merNum = 0; + SFilterRangeCtx *ctx = NULL; + SFilterColCtx colCtx = {0}; + SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx)); + + for (; m < (*gRes1)->colNum; ++m) { + idx1 = (*gRes1)->colIdx[m]; + + for (; n < (*gRes2)->colNum; ++n) { + idx2 = (*gRes2)->colIdx[n]; + + if (idx1 > idx2) { + continue; + } + + assert(idx1 == idx2); + + ++merNum; + + filterMergeTwoGroupsImpl(info, &ctx, TSDB_RELATION_OR, idx1, *gRes1, *gRes2, NULL, all); + + CHK_JMP(*all); + + if (numEqual) { + if ((*gRes1)->colNum == 1) { + ++equal1; + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + taosArrayPush(colCtxs, &colCtx); + break; + } else { + filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); + if (equal) { + ++equal1; + } + + filterCompareRangeCtx(ctx, (*gRes2)->colInfo[idx2].info, &equal); + if (equal) { + ++equal2; + } + + CHK_JMP(equal1 != merNum && equal2 != merNum); + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + ctx = NULL; + taosArrayPush(colCtxs, &colCtx); + } + } else { + filterCompareRangeCtx(ctx, (*gRes1)->colInfo[idx1].info, &equal); + if (equal) { + ++equal1; + } + + CHK_JMP(equal1 != merNum); + colCtx.colIdx = idx1; + colCtx.ctx = ctx; + ctx = NULL; + taosArrayPush(colCtxs, &colCtx); + } + + ++n; + break; + } + } + + assert(merNum > 0); + + SFilterColInfo *colInfo = NULL; + assert (merNum == equal1 || merNum == equal2); + + filterFreeGroupCtx(*gRes2); + *gRes2 = NULL; + + assert(colCtxs && taosArrayGetSize(colCtxs) > 0); + + int32_t ctxSize = (int32_t)taosArrayGetSize(colCtxs); + SFilterColCtx *pctx = NULL; + + for (int32_t i = 0; i < ctxSize; ++i) { + pctx = taosArrayGet(colCtxs, i); + colInfo = &(*gRes1)->colInfo[pctx->colIdx]; + + filterFreeColInfo(colInfo); + FILTER_PUSH_CTX((*gRes1)->colInfo[pctx->colIdx], pctx->ctx); + } + + taosArrayDestroy(colCtxs); + + return TSDB_CODE_SUCCESS; + +_return: + + if (colCtxs) { + if (taosArrayGetSize(colCtxs) > 0) { + taosArrayDestroyEx(colCtxs, filterFreeColCtx); + } else { + taosArrayDestroy(colCtxs); + } + } + + filterFreeRangeCtx(ctx); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gResNum) { + if (*gResNum <= 1) { + return TSDB_CODE_SUCCESS; + } + + qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx); + + int32_t pEnd = 0, cStart = 0, cEnd = 0; + uint16_t pColNum = 0, cColNum = 0; + int32_t movedNum = 0; + bool all = false; + + cColNum = gRes[0]->colNum; + + for (int32_t i = 1; i <= *gResNum; ++i) { + if (i < (*gResNum) && gRes[i]->colNum == cColNum) { + continue; + } + + cEnd = i - 1; + + movedNum = 0; + if (pColNum > 0) { + for (int32_t m = 0; m <= pEnd; ++m) { + for (int32_t n = cStart; n <= cEnd; ++n) { + assert(m < n); + filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); + + CHK_JMP(all); + + if (gRes[n] == NULL) { + if (n < ((*gResNum) - 1)) { + memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); + } + + --cEnd; + --(*gResNum); + ++movedNum; + --n; + } + } + } + } + + for (int32_t m = cStart; m < cEnd; ++m) { + for (int32_t n = m + 1; n <= cEnd; ++n) { + assert(m < n); + filterMergeTwoGroups(info, &gRes[m], &gRes[n], &all); + + CHK_JMP(all); + + if (gRes[n] == NULL) { + if (n < ((*gResNum) - 1)) { + memmove(&gRes[n], &gRes[n+1], (*gResNum-n-1) * POINTER_BYTES); + } + + --cEnd; + --(*gResNum); + ++movedNum; + --n; + } + } + } + + pColNum = cColNum; + pEnd = cEnd; + + i -= movedNum; + + if (i >= (*gResNum)) { + break; + } + + cStart = i; + cEnd = i; + cColNum = gRes[i]->colNum; + } + + return TSDB_CODE_SUCCESS; + +_return: + + FILTER_SET_FLAG(info->status, FI_STATUS_ALL); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) { + size_t groupSize = taosArrayGetSize(group); + + info->groupNum = (uint16_t)groupSize; + + if (info->groupNum > 0) { + info->groups = calloc(info->groupNum, sizeof(*info->groups)); + } + + for (size_t i = 0; i < groupSize; ++i) { + SFilterGroup *pg = taosArrayGet(group, i); + pg->unitFlags = calloc(pg->unitNum, sizeof(*pg->unitFlags)); + info->groups[i] = *pg; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { + if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) { + //qDebug("no need rewrite"); + return TSDB_CODE_SUCCESS; + } + + SFilterInfo oinfo = *info; + + FILTER_SET_FLAG(oinfo.status, FI_STATUS_CLONED); + + SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + SFilterGroupCtx *res = NULL; + SFilterColInfo *colInfo = NULL; + int32_t optr = 0; + uint16_t uidx = 0; + + memset(info, 0, sizeof(*info)); + + info->colRangeNum = oinfo.colRangeNum; + info->colRange = oinfo.colRange; + oinfo.colRangeNum = 0; + oinfo.colRange = NULL; + + FILTER_SET_FLAG(info->options, FI_OPTION_NEED_UNIQE); + + filterInitUnitsFields(info); + + for (int32_t i = 0; i < gResNum; ++i) { + res = gRes[i]; + + optr = (res->colNum > 1) ? TSDB_RELATION_AND : TSDB_RELATION_OR; + + SFilterGroup ng = {0}; + + for (uint16_t m = 0; m < res->colNum; ++m) { + colInfo = &res->colInfo[res->colIdx[m]]; + if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { + assert(colInfo->type == RANGE_TYPE_UNIT); + int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); + + for (int32_t n = 0; n < usize; ++n) { + SFilterUnit* u = taosArrayGetP((SArray *)colInfo->info, n); + + filterAddUnitFromUnit(info, &oinfo, u, &uidx); + filterAddUnitToGroup(&ng, uidx); + } + + continue; + } + + assert(colInfo->type == RANGE_TYPE_MR_CTX); + + filterAddGroupUnitFromCtx(info, &oinfo, colInfo->info, res->colIdx[m], &ng, optr, group); + } + + if (ng.unitNum > 0) { + taosArrayPush(group, &ng); + } + } + + filterConvertGroupFromArray(info, group); + + taosArrayDestroy(group); + + filterFreeInfo(&oinfo); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) { + uint16_t *idxs = NULL; + uint16_t colNum = 0; + SFilterGroupCtx *res = NULL; + uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum)); + + for (int32_t i = 0; i < gResNum; ++i) { + for (uint16_t m = 0; m < gRes[i]->colNum; ++m) { + SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]]; + if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) { + continue; + } + + ++idxNum[gRes[i]->colIdx[m]]; + } + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + if (idxNum[i] < gResNum) { + continue; + } + + assert(idxNum[i] == gResNum); + + if (idxs == NULL) { + idxs = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxs)); + } + + idxs[colNum++] = i; + } + + CHK_JMP(colNum <= 0); + + info->colRangeNum = colNum; + info->colRange = calloc(colNum, POINTER_BYTES); + + for (int32_t i = 0; i < gResNum; ++i) { + res = gRes[i]; + uint16_t n = 0; + + for (uint16_t m = 0; m < info->colRangeNum; ++m) { + for (; n < res->colNum; ++n) { + if (res->colIdx[n] < idxs[m]) { + continue; + } + + assert(res->colIdx[n] == idxs[m]); + + SFilterColInfo * colInfo = &res->colInfo[res->colIdx[n]]; + if (info->colRange[m] == NULL) { + info->colRange[m] = filterInitRangeCtx(colInfo->dataType, 0); + SFilterField* fi = FILTER_GET_COL_FIELD(info, res->colIdx[n]); + info->colRange[m]->colId = ((SSchema*)fi->desc)->colId; + } + + assert(colInfo->type == RANGE_TYPE_MR_CTX); + + bool all = false; + filterSourceRangeFromCtx(info->colRange[m], colInfo->info, TSDB_RELATION_OR, NULL, &all); + if (all) { + filterFreeRangeCtx(info->colRange[m]); + info->colRange[m] = NULL; + + if (m < (info->colRangeNum - 1)) { + memmove(&info->colRange[m], &info->colRange[m + 1], (info->colRangeNum - m - 1) * POINTER_BYTES); + memmove(&idxs[m], &idxs[m + 1], (info->colRangeNum - m - 1) * sizeof(*idxs)); + } + + --info->colRangeNum; + --m; + + CHK_JMP(info->colRangeNum <= 0); + } + + ++n; + break; + } + } + } + +_return: + tfree(idxNum); + tfree(idxs); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterPostProcessRange(SFilterInfo *info) { + for (uint16_t i = 0; i < info->colRangeNum; ++i) { + SFilterRangeCtx* ctx = info->colRange[i]; + SFilterRangeNode *r = ctx->rs; + while (r) { + r->rc.func = filterGetRangeCompFunc(r->ra.sflag, r->ra.eflag); + r = r->next; + } + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterGenerateComInfo(SFilterInfo *info) { + uint16_t n = 0; + + info->cunits = malloc(info->unitNum * sizeof(*info->cunits)); + info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum); + info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum); + + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + + info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr); + info->cunits[i].rfunc = filterGetRangeCompFuncFromOptrs(unit->compare.optr, unit->compare.optr2); + info->cunits[i].optr = FILTER_UNIT_OPTR(unit); + info->cunits[i].colData = NULL; + info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit); + + if (unit->right.type == FLD_TYPE_VALUE) { + info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit); + } else { + info->cunits[i].valData = NULL; + } + if (unit->right2.type == FLD_TYPE_VALUE) { + info->cunits[i].valData2 = FILTER_GET_VAL_FIELD_DATA(FILTER_GET_FIELD(info, unit->right2)); + } else { + info->cunits[i].valData2 = info->cunits[i].valData; + } + + info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit); + info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit); + } + + uint16_t cgroupNum = info->groupNum + 1; + + for (uint16_t i = 0; i < info->groupNum; ++i) { + cgroupNum += info->groups[i].unitNum; + } + + info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups)); + + for (uint16_t i = 0; i < info->groupNum; ++i) { + info->cgroups[n++] = info->groups[i].unitNum; + + for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) { + info->cgroups[n++] = info->groups[i].unitIdxs[m]; + } + } + + info->cgroups[n] = 0; + + return TSDB_CODE_SUCCESS; +} + +int32_t filterUpdateComUnits(SFilterInfo *info) { + for (uint16_t i = 0; i < info->unitNum; ++i) { + SFilterUnit *unit = &info->units[i]; + + info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { + int32_t rmUnit = 0; + + memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum); + + for (int32_t k = 0; k < info->unitNum; ++k) { + int32_t index = -1; + SFilterComUnit *cunit = &info->cunits[k]; + + if (FILTER_NO_MERGE_DATA_TYPE(cunit->dataType)) { + continue; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + if (pDataStatis[i].colId == cunit->colId) { + index = i; + break; + } + } + + if (index == -1) { + continue; + } + + if (pDataStatis[index].numOfNull <= 0) { + if (cunit->optr == TSDB_RELATION_ISNULL) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + continue; + } + + if (cunit->optr == TSDB_RELATION_NOTNULL) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + continue; + } + } else { + if (pDataStatis[index].numOfNull == numOfRows) { + if (cunit->optr == TSDB_RELATION_ISNULL) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + continue; + } + } + + if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL + || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH + || cunit->optr == TSDB_RELATION_NOT_EQUAL) { + continue; + } + + SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + void *minVal, *maxVal; + + if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) { + float minv = (float)(*(double *)(&pDataBlockst->min)); + float maxv = (float)(*(double *)(&pDataBlockst->max)); + + minVal = &minv; + maxVal = &maxv; + } else { + minVal = &pDataBlockst->min; + maxVal = &pDataBlockst->max; + } + + bool minRes = false, maxRes = false; + + if (cunit->rfunc >= 0) { + minRes = (*gRangeCompare[cunit->rfunc])(minVal, minVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + maxRes = (*gRangeCompare[cunit->rfunc])(maxVal, maxVal, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS_EQUAL, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER_EQUAL, maxVal, cunit->valData2); + + if (minRes && maxRes) { + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + } else { + minRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], cunit->optr, maxVal, cunit->valData); + + if (minRes && maxRes) { + info->blkUnitRes[k] = 1; + rmUnit = 1; + } else if ((!minRes) && (!maxRes)) { + if (cunit->optr == TSDB_RELATION_EQUAL) { + minRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_GREATER, minVal, cunit->valData); + maxRes = filterDoCompare(gDataCompare[cunit->func], TSDB_RELATION_LESS, maxVal, cunit->valData); + if (minRes || maxRes) { + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + + continue; + } + + info->blkUnitRes[k] = -1; + rmUnit = 1; + } + } + + } + +// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY"); + + info->blkGroupNum = info->groupNum; + + uint16_t *unitNum = info->blkUnits; + uint16_t *unitIdx = unitNum + 1; + int32_t all = 0, empty = 0; + + for (uint32_t g = 0; g < info->groupNum; ++g) { + SFilterGroup *group = &info->groups[g]; + *unitNum = group->unitNum; + all = 0; + empty = 0; + + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + if (info->blkUnitRes[uidx] == 1) { + --(*unitNum); + all = 1; + continue; + } else if (info->blkUnitRes[uidx] == -1) { + *unitNum = 0; + empty = 1; + break; + } + + *(unitIdx++) = uidx; + } + + if (*unitNum == 0) { + --info->blkGroupNum; + assert(empty || all); + + if (empty) { + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); + } else { + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL); + goto _return; + } + + continue; + } + + unitNum = unitIdx; + ++unitIdx; + } + + if (info->blkGroupNum) { + FILTER_CLR_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY); + FILTER_SET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE); + } + +_return: + + filterDumpInfoToString(info, "Block Filter", 2); + + return TSDB_CODE_SUCCESS; +} + +bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + uint16_t *unitIdx = NULL; + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + //FILTER_UNIT_CLR_F(info); + + unitIdx = info->blkUnits; + + for (uint32_t g = 0; g < info->blkGroupNum; ++g) { + uint16_t unitNum = *(unitIdx++); + for (uint32_t u = 0; u < unitNum; ++u) { + SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; + void *colData = (char *)cunit->colData + cunit->dataSize * i; + + //if (FILTER_UNIT_GET_F(info, uidx)) { + // p[i] = FILTER_UNIT_GET_R(info, uidx); + //} else { + uint8_t optr = cunit->optr; + + if (isNull(colData, cunit->dataType)) { + (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + } else { + if (optr == TSDB_RELATION_NOTNULL) { + (*p)[i] = 1; + } else if (optr == TSDB_RELATION_ISNULL) { + (*p)[i] = 0; + } else if (cunit->rfunc >= 0) { + (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } + + //FILTER_UNIT_SET_R(info, uidx, p[i]); + //FILTER_UNIT_SET_F(info, uidx); + } + + if ((*p)[i] == 0) { + break; + } + } + + if ((*p)[i]) { + break; + } + + unitIdx += unitNum; + } + + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + + + +int32_t filterExecuteBasedOnStatis(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols, bool* all) { + if (statis && numOfRows >= FILTER_RM_UNIT_MIN_ROWS) { + info->blkFlag = 0; + + filterRmUnitByRange(info, statis, numOfCols, numOfRows); + + if (info->blkFlag) { + if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) { + *all = true; + goto _return; + } else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) { + *all = false; + goto _return; + } + + assert(info->unitNum > 1); + + *all = filterExecuteBasedOnStatisImpl(info, numOfRows, p, statis, numOfCols); + + goto _return; + } + } + + return 1; + +_return: + info->blkFlag = 0; + + return TSDB_CODE_SUCCESS; +} + + +static FORCE_INLINE bool filterExecuteImplAll(void *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + return true; +} +static FORCE_INLINE bool filterExecuteImplEmpty(void *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + return false; +} +static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} +static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + +bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + uint16_t dataSize = info->cunits[0].dataSize; + char *colData = (char *)info->cunits[0].colData; + rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; + void *valData = info->cunits[0].valData; + void *valData2 = info->cunits[0].valData2; + __compar_fn_t func = gDataCompare[info->cunits[0].func]; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + if (isNull(colData, info->cunits[0].dataType)) { + all = false; + colData += dataSize; + continue; + } + + (*p)[i] = (*rfunc)(colData, colData, valData, valData2, func); + + if ((*p)[i] == 0) { + all = false; + } + + colData += dataSize; + } + + return all; +} + +bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + uint16_t uidx = info->groups[0].unitIdxs[0]; + void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + if (isNull(colData, info->cunits[uidx].dataType)) { + all = false; + continue; + } + + (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData); + + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + + +bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + SFilterInfo *info = (SFilterInfo *)pinfo; + bool all = true; + + if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) { + return all; + } + + *p = calloc(numOfRows, sizeof(int8_t)); + + for (int32_t i = 0; i < numOfRows; ++i) { + //FILTER_UNIT_CLR_F(info); + + for (uint32_t g = 0; g < info->groupNum; ++g) { + SFilterGroup *group = &info->groups[g]; + for (uint32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + SFilterComUnit *cunit = &info->cunits[uidx]; + void *colData = (char *)cunit->colData + cunit->dataSize * i; + + //if (FILTER_UNIT_GET_F(info, uidx)) { + // p[i] = FILTER_UNIT_GET_R(info, uidx); + //} else { + uint8_t optr = cunit->optr; + + if (isNull(colData, cunit->dataType)) { + (*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false; + } else { + if (optr == TSDB_RELATION_NOTNULL) { + (*p)[i] = 1; + } else if (optr == TSDB_RELATION_ISNULL) { + (*p)[i] = 0; + } else if (cunit->rfunc >= 0) { + (*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]); + } else { + (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData); + } + + //FILTER_UNIT_SET_R(info, uidx, p[i]); + //FILTER_UNIT_SET_F(info, uidx); + } + + if ((*p)[i] == 0) { + break; + } + } + + if ((*p)[i]) { + break; + } + } + + if ((*p)[i] == 0) { + all = false; + } + } + + return all; +} + + +FORCE_INLINE bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { + return (*info->func)(info, numOfRows, p, statis, numOfCols); +} + +int32_t filterSetExecFunc(SFilterInfo *info) { + if (FILTER_ALL_RES(info)) { + info->func = filterExecuteImplAll; + return TSDB_CODE_SUCCESS; + } + + if (FILTER_EMPTY_RES(info)) { + info->func = filterExecuteImplEmpty; + return TSDB_CODE_SUCCESS; + } + + if (info->unitNum > 1) { + info->func = filterExecuteImpl; + return TSDB_CODE_SUCCESS; + } + + if (info->units[0].compare.optr == TSDB_RELATION_ISNULL) { + info->func = filterExecuteImplIsNull; + return TSDB_CODE_SUCCESS; + } + + if (info->units[0].compare.optr == TSDB_RELATION_NOTNULL) { + info->func = filterExecuteImplNotNull; + return TSDB_CODE_SUCCESS; + } + + if (info->cunits[0].rfunc >= 0) { + info->func = filterExecuteImplRange; + return TSDB_CODE_SUCCESS; + } + + info->func = filterExecuteImplMisc; + return TSDB_CODE_SUCCESS; +} + + + +int32_t filterPreprocess(SFilterInfo *info) { + SFilterGroupCtx** gRes = calloc(info->groupNum, sizeof(SFilterGroupCtx *)); + int32_t gResNum = 0; + + filterMergeGroupUnits(info, gRes, &gResNum); + + filterMergeGroups(info, gRes, &gResNum); + + if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { +// qInfo("Final - FilterInfo: [ALL]"); + goto _return; + } + + + if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { +// qInfo("Final - FilterInfo: [EMPTY]"); + goto _return; + } + + filterGenerateColRange(info, gRes, gResNum); + + filterDumpInfoToString(info, "Final", 1); + + filterPostProcessRange(info); + + filterRewrite(info, gRes, gResNum); + + filterGenerateComInfo(info); + +_return: + + filterSetExecFunc(info); + + for (int32_t i = 0; i < gResNum; ++i) { + filterFreeGroupCtx(gRes[i]); + } + + tfree(gRes); + + return TSDB_CODE_SUCCESS; +} + +int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) { +// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); +// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); + + if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema* sch = fi->desc; + + for (int32_t j = 0; j < numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); + if (sch->colId == pColInfo->info.colId) { + fi->data = pColInfo->pData; + + break; + } + } + } + + filterUpdateComUnits(info); + + return TSDB_CODE_SUCCESS; +} + + +int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) { + int32_t code = TSDB_CODE_SUCCESS; + SFilterInfo *info = NULL; + +// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param"); + + if (*pinfo == NULL) { + *pinfo = calloc(1, sizeof(SFilterInfo)); + } + + info = *pinfo; + + info->options = options; + + SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup)); + + filterInitUnitsFields(info); + + code = filterTreeToGroup(tree, info, group); + + ERR_JRET(code); + + filterConvertGroupFromArray(info, group); + + ERR_JRET(filterInitValFieldData(info)); + + if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) { + filterDumpInfoToString(info, "Before preprocess", 0); + + ERR_JRET(filterPreprocess(info)); + + CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL)); + + if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { + taosArrayDestroy(group); + return code; + } + + //ERR_JRET(filterInitUnitFunc(info)); + } + + info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes)); + info->unitFlags = malloc(info->unitNum * sizeof(*info->unitFlags)); + + filterDumpInfoToString(info, "Final", 0); + + taosArrayDestroy(group); + + return code; + +_return: +// qInfo("No filter, code:%d", code); + + taosArrayDestroy(group); + filterFreeInfo(*pinfo); + + *pinfo = NULL; + + return code; +} + + + + +bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { + if (FILTER_EMPTY_RES(info)) { + return false; + } + + if (FILTER_ALL_RES(info)) { + return true; + } + + bool ret = true; + void *minVal, *maxVal; + + for (int32_t k = 0; k < info->colRangeNum; ++k) { + int32_t index = -1; + SFilterRangeCtx *ctx = info->colRange[k]; + for(int32_t i = 0; i < numOfCols; ++i) { + if (pDataStatis[i].colId == ctx->colId) { + index = i; + break; + } + } + + // no statistics data, load the true data block + if (index == -1) { + break; + } + + // not support pre-filter operation on binary/nchar data type + if (FILTER_NO_MERGE_DATA_TYPE(ctx->type)) { + break; + } + + if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) { + ret = false; + break; + } + + // all data in current column are NULL, no need to check its boundary value + if (pDataStatis[index].numOfNull == numOfRows) { + + // if isNULL query exists, load the null data column + if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { + ret = false; + break; + } + + continue; + } + + SColumnDataAgg* pDataBlockst = &pDataStatis[index]; + + SFilterRangeNode *r = ctx->rs; + + if (ctx->type == TSDB_DATA_TYPE_FLOAT) { + float minv = (float)(*(double *)(&pDataBlockst->min)); + float maxv = (float)(*(double *)(&pDataBlockst->max)); + + minVal = &minv; + maxVal = &maxv; + } else { + minVal = &pDataBlockst->min; + maxVal = &pDataBlockst->max; + } + + while (r) { + ret = r->rc.func(minVal, maxVal, &r->rc.s, &r->rc.e, ctx->pCompareFunc); + if (ret) { + break; + } + r = r->next; + } + + CHK_RET(!ret, ret); + } + + return ret; +} + + + +int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) { + SFilterRange ra = {0}; + SFilterRangeCtx *prev = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *tmpc = filterInitRangeCtx(TSDB_DATA_TYPE_TIMESTAMP, FI_OPTION_TIMESTAMP); + SFilterRangeCtx *cur = NULL; + int32_t num = 0; + int32_t optr = 0; + int32_t code = 0; + bool empty = false, all = false; + + for (int32_t i = 0; i < info->groupNum; ++i) { + SFilterGroup *group = &info->groups[i]; + if (group->unitNum > 1) { + cur = tmpc; + optr = TSDB_RELATION_AND; + } else { + cur = prev; + optr = TSDB_RELATION_OR; + } + + for (int32_t u = 0; u < group->unitNum; ++u) { + uint16_t uidx = group->unitIdxs[u]; + SFilterUnit *unit = &info->units[uidx]; + + uint8_t raOptr = FILTER_UNIT_OPTR(unit); + + filterAddRangeOptr(cur, raOptr, TSDB_RELATION_AND, &empty, NULL); + CHK_JMP(empty); + + if (FILTER_NO_MERGE_OPTR(raOptr)) { + continue; + } + + SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); + void *s = FILTER_GET_VAL_FIELD_DATA(right); + void *e = FILTER_GET_VAL_FIELD_DATA(right) + tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + + SIMPLE_COPY_VALUES(&ra.s, s); + SIMPLE_COPY_VALUES(&ra.e, e); + + filterAddRange(cur, &ra, optr); + } + + if (cur->notnull) { + prev->notnull = true; + break; + } + + if (group->unitNum > 1) { + filterSourceRangeFromCtx(prev, cur, TSDB_RELATION_OR, &empty, &all); + filterResetRangeCtx(cur); + if (all) { + break; + } + } + } + + if (prev->notnull) { + *win = TSWINDOW_INITIALIZER; + } else { + filterGetRangeNum(prev, &num); + if (num > 1) { + //qError("only one time range accepted, num:%d", num); + ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION); + } + + CHK_JMP(num < 1); + + SFilterRange tra; + filterGetRangeRes(prev, &tra); + win->skey = tra.s; + win->ekey = tra.e; + } + + filterFreeRangeCtx(prev); + filterFreeRangeCtx(tmpc); + + //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + return TSDB_CODE_SUCCESS; + +_return: + + *win = TSWINDOW_DESC_INITIALIZER; + + filterFreeRangeCtx(prev); + filterFreeRangeCtx(tmpc); + + //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey); + + return code; +} + + +int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) { + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + if (type == TSDB_DATA_TYPE_NCHAR) { + SFilterField nfi = {0}; + nfi.desc = fi->desc; + int32_t bytes = FILTER_GET_COL_FIELD_SIZE(fi); + nfi.data = malloc(rows * bytes); + int32_t bufSize = bytes - VARSTR_HEADER_SIZE; + for (int32_t j = 0; j < rows; ++j) { + char *src = FILTER_GET_COL_FIELD_DATA(fi, j); + char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j); + int32_t len = 0; + taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len); + varDataLen(dst) = len; + } + + fi->data = nfi.data; + + *gotNchar = true; + } + } + + if (*gotNchar) { + filterUpdateComUnits(info); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t filterFreeNcharColumns(SFilterInfo* info) { + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + int32_t type = FILTER_GET_COL_FIELD_TYPE(fi); + if (type == TSDB_DATA_TYPE_NCHAR) { + tfree(fi->data); + } + } + + return TSDB_CODE_SUCCESS; +} + + + + + diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c5e7ea12a5..c3bc63cf6f 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -30,12 +30,12 @@ extern "C" { extern SAggFunctionInfo aggFunc[34]; -typedef struct SResultRowCellInfo { +typedef struct SResultRowEntryInfo { int8_t hasResult; // result generated, not NULL value bool initialized; // output buffer has been initialized bool complete; // query has completed uint32_t numOfRes; // num of output result in current buffer -} SResultRowCellInfo; +} SResultRowEntryInfo; #define FUNCSTATE_SO 0x0u #define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM @@ -52,54 +52,24 @@ typedef struct SResultRowCellInfo { #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) -#define QUERY_ASC_FORWARD_STEP 1 -#define QUERY_DESC_FORWARD_STEP -1 - -#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define TOP_BOTTOM_QUERY_LIMIT 100 -enum { - MASTER_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - MERGE_STAGE = 0x20u, -}; - #define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0) #define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY)) #define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) #define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0) -typedef struct SArithmeticSupport { - struct SExprInfo *pExprInfo; - int32_t numOfCols; - SColumnInfo *colList; - void *exprList; // client side used - int32_t offset; - char** data; -} SArithmeticSupport; - typedef struct SInterpInfoDetail { TSKEY ts; // interp specified timestamp int8_t type; int8_t primaryCol; } SInterpInfoDetail; -#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) - -#define GET_RES_INFO(ctx) ((ctx)->resultInfo) +#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) #define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) -// determine the real data need to calculated the result -enum { - BLK_DATA_NO_NEEDED = 0x0, - BLK_DATA_STATIS_NEEDED = 0x1, - BLK_DATA_ALL_NEEDED = 0x3, - BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter -}; - typedef struct STwaInfo { int8_t hasResult; // flag to denote has value double dOutput; @@ -115,12 +85,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha * the numOfRes should be kept, since it may be used later * and allow the ResultInfo to be re initialized */ -#define RESET_RESULT_INFO(_r) \ - do { \ - (_r)->initialized = false; \ - } while (0) - -static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) { +static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; diff --git a/source/libs/function/inc/texpr.h b/source/libs/function/inc/texpr.h index bb5c72f7d1..4ef0a7ab21 100644 --- a/source/libs/function/inc/texpr.h +++ b/source/libs/function/inc/texpr.h @@ -60,7 +60,6 @@ typedef struct SExprTraverseSupp { void *pExtInfo; } SExprTraverseSupp; -tExprNode* exprTreeFromBinary(const void* data, size_t size); tExprNode* exprTreeFromTableName(const char* tbnameCond); bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); diff --git a/source/libs/function/inc/tfill.h b/source/libs/function/inc/tfill.h index 978feb001d..81348fba1d 100644 --- a/source/libs/function/inc/tfill.h +++ b/source/libs/function/inc/tfill.h @@ -25,7 +25,7 @@ extern "C" { struct SSDataBlock; -typedef struct { +typedef struct SFillColInfo { STColumn col; // column info int16_t functionId; // sql function id int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN @@ -64,30 +64,11 @@ typedef struct SFillInfo { void* handle; // for debug purpose } SFillInfo; -typedef struct SPoint { - int64_t key; - void * val; -} SPoint; - -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pFillCol, void* handle); - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); - -void* taosDestroyFillInfo(SFillInfo *pFillInfo); - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); - -void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); - -bool taosFillHasMoreResults(SFillInfo* pFillInfo); - int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity); + + #ifdef __cplusplus } diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/function/inc/tscalarfunction.h index eafec75d87..d8e4c1eeaf 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/function/inc/tscalarfunction.h @@ -28,16 +28,22 @@ typedef struct SScalarFuncParam { int32_t bytes; } SScalarFuncParam; -extern struct SScalarFunctionInfo scalarFunc[1]; +typedef struct SScalarFunctionSupport { + struct SExprInfo *pExprInfo; + int32_t numOfCols; + SColumnInfo *colList; + void *exprList; // client side used + int32_t offset; + char** data; +} SScalarFunctionSupport; -#define FUNCTION_CEIL 38 -#define FUNCTION_FLOOR 39 -#define FUNCTION_ROUND 40 -#define FUNCTION_CONCAT 41 +extern struct SScalarFunctionInfo scalarFunc[1]; int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); + + #ifdef __cplusplus } #endif diff --git a/src/query/inc/qScript.h b/source/libs/function/inc/tscript.h similarity index 98% rename from src/query/inc/qScript.h rename to source/libs/function/inc/tscript.h index 574bb51368..281fe6f679 100644 --- a/src/query/inc/qScript.h +++ b/source/libs/function/inc/tscript.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_QSCRIPT_H #define TDENGINE_QSCRIPT_H +#if 0 #include #include #include @@ -23,7 +24,7 @@ #include "tutil.h" #include "hash.h" #include "tlist.h" -#include "qUdf.h" +#include "tudf.h" #define MAX_FUNC_NAME 64 @@ -78,5 +79,6 @@ void destroyScriptCtx(void *pScriptCtx); int32_t scriptEnvPoolInit(); void scriptEnvPoolCleanup(); bool isValidScript(char *script, int32_t len); +#endif #endif //TDENGINE_QSCRIPT_H diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index dc643ace9e..163fbdf4bb 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -16,6 +16,13 @@ #ifndef TDENGINE_TUDF_H #define TDENGINE_TUDF_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "taoserror.h" + enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, @@ -76,4 +83,8 @@ typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOf typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf); typedef void (*udfDestroyFunc)(SUdfInit* buf); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_TUDF_H diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 5d3f93f9d6..f4e48f2faa 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ +#include "tscalarfunction.h" #include "os.h" #include "taosdef.h" #include "taosmsg.h" -//#include "texpr.h" -#include "ttypes.h" #include "tglobal.h" #include "thash.h" +#include "ttypes.h" #include "taggfunction.h" #include "tfill.h" @@ -78,7 +78,7 @@ void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } +void doFinalizer(SQLFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } typedef struct tValuePair { SVariant v; @@ -196,6 +196,49 @@ typedef struct SFileBlockInfo { int32_t numBlocksOfStep; } SFileBlockInfo; +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { + pCell->initialized = false; +} + +int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { + int32_t maxOutput = 0; + for (int32_t j = 0; j < num; ++j) { + int32_t id = pCtx[j].functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (/*hasMainFunction && */(id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ)) { + continue; + } + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { + maxOutput = pResInfo->numOfRes; + } + } + + assert(maxOutput >= 0); + return maxOutput; +} + +void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num) { + for (int32_t j = 0; j < num; ++j) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + pResInfo->numOfRes = 0; + } +} + +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { + assert(pEntry != NULL); + return pEntry->complete; +} + +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { + return pEntry->initialized; +} + int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { if (!isValidDataType(dataType)) { @@ -430,13 +473,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; } memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResultInfo, pCtx->interBufBytes); + initResultRowEntry(pResultInfo, pCtx->interBufBytes); return true; } @@ -448,7 +491,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo * @param pCtx */ static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } @@ -464,12 +507,12 @@ static void count_function(SQLFunctionCtx *pCtx) { int32_t numOfElem = 0; /* - * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isSmaSet == true; - * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isSmaSet == true; - * 3. for primary key column, pCtx->hasNull always be false, pCtx->isSmaSet == false; + * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isAggSet == true; + * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isAggSet == true; + * 3. for primary key column, pCtx->hasNull always be false, pCtx->isAggSet == false; */ - if (pCtx->isSmaSet) { - numOfElem = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { + numOfElem = pCtx->size - pCtx->agg.numOfNull; } else { if (pCtx->hasNull) { for (int32_t i = 0; i < pCtx->size; ++i) { @@ -596,19 +639,19 @@ static void do_sum(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // Only the pre-computing information loaded and actual data does not loaded - if (pCtx->isSmaSet) { - notNullElems = pCtx->size - pCtx->sma.numOfNull; - assert(pCtx->size >= pCtx->sma.numOfNull); + if (pCtx->isAggSet) { + notNullElems = pCtx->size - pCtx->agg.numOfNull; + assert(pCtx->size >= pCtx->agg.numOfNull); if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { int64_t *retVal = (int64_t *)pCtx->pOutput; - *retVal += pCtx->sma.sum; + *retVal += pCtx->agg.sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { uint64_t *retVal = (uint64_t *)pCtx->pOutput; - *retVal += (uint64_t)pCtx->sma.sum; + *retVal += (uint64_t)pCtx->agg.sum; } else if (IS_FLOAT_TYPE(pCtx->inputType)) { double *retVal = (double*) pCtx->pOutput; - SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->sma.sum))); + SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->agg.sum))); } } else { // computing based on the true data block void *pData = GET_INPUT_DATA_LIST(pCtx); @@ -659,7 +702,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; @@ -692,7 +735,7 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } SET_VAL(pCtx, notNullElems, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -783,21 +826,21 @@ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); double *pVal = &pAvgInfo->sum; - if (pCtx->isSmaSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { // Pre-aggregation + notNullElems = pCtx->size - pCtx->agg.numOfNull; assert(notNullElems >= 0); if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->sma.sum; + *pVal += pCtx->agg.sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->sma.sum; + *pVal += (uint64_t) pCtx->agg.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->sma.sum)); + *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->agg.sum)); } } else { void *pData = GET_INPUT_DATA_LIST(pCtx); @@ -843,7 +886,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->pOutput; char *input = GET_INPUT_DATA_LIST(pCtx); @@ -865,7 +908,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -897,8 +940,8 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { // data in current data block are qualified to the query - if (pCtx->isSmaSet) { - *notNullElems = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { + *notNullElems = pCtx->size - pCtx->agg.numOfNull; assert(*notNullElems >= 0); if (*notNullElems == 0) { @@ -909,11 +952,11 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int16_t index = 0; if (isMin) { - tval = &pCtx->sma.min; - index = pCtx->sma.minIndex; + tval = &pCtx->agg.min; + index = pCtx->agg.minIndex; } else { - tval = &pCtx->sma.max; - index = pCtx->sma.maxIndex; + tval = &pCtx->agg.max; + index = pCtx->agg.maxIndex; } TSKEY key = TSKEY_INITIAL_VAL; @@ -1046,7 +1089,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } } -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1092,7 +1135,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo return true; } -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1148,7 +1191,7 @@ static void min_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query @@ -1165,7 +1208,7 @@ static void max_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query @@ -1265,7 +1308,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1276,7 +1319,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1292,7 +1335,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { } static void stddev_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { @@ -1494,7 +1537,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { } static void stddev_dst_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); char *input = GET_INPUT_DATA_LIST(pCtx); @@ -1525,7 +1568,7 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -1558,7 +1601,7 @@ static void first_function(SQLFunctionCtx *pCtx) { DO_UPDATE_TAG_COLUMNS(pCtx, k); } - SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; @@ -1608,7 +1651,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { first_data_assign_impl(pCtx, data, i); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1653,7 +1696,7 @@ static void last_function(SQLFunctionCtx *pCtx) { return; } - SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; if (pCtx->order == TSDB_ORDER_DESC) { @@ -1738,7 +1781,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { last_data_assign_impl(pCtx, data, i); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1788,7 +1831,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // assign the last element in current data block assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query @@ -1808,7 +1851,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; @@ -1981,7 +2024,7 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); tValuePair **tvp = pRes->res; @@ -2076,7 +2119,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * top/bottom use the intermediate result buffer to keep the intermediate result */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { @@ -2108,7 +2151,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { } bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; } @@ -2163,7 +2206,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha } } -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2204,7 +2247,7 @@ static void top_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2227,7 +2270,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2261,7 +2304,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2284,13 +2327,13 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -2318,7 +2361,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { } /////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2335,7 +2378,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { @@ -2353,17 +2396,17 @@ static void percentile_function(SQLFunctionCtx *pCtx) { // the first stage, only acquire the min/max value if (pInfo->stage == 0) { - if (pCtx->isSmaSet) { + if (pCtx->isAggSet) { double tmin = 0.0, tmax = 0.0; if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->sma.min); - tmax = (double)GET_INT64_VAL(&pCtx->sma.max); + tmin = (double)GET_INT64_VAL(&pCtx->agg.min); + tmax = (double)GET_INT64_VAL(&pCtx->agg.max); } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->sma.min); - tmax = GET_DOUBLE_VAL(&pCtx->sma.max); + tmin = GET_DOUBLE_VAL(&pCtx->agg.min); + tmax = GET_DOUBLE_VAL(&pCtx->agg.max); } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->sma.min); - tmax = (double)GET_UINT64_VAL(&pCtx->sma.max); + tmin = (double)GET_UINT64_VAL(&pCtx->agg.min); + tmax = (double)GET_UINT64_VAL(&pCtx->agg.max); } else { assert(true); } @@ -2376,7 +2419,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SET_DOUBLE_VAL(&pInfo->maxval, tmax); } - pInfo->numOfElems += (pCtx->size - pCtx->sma.numOfNull); + pInfo->numOfElems += (pCtx->size - pCtx->agg.numOfNull); } else { for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2420,7 +2463,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); tMemBucket * pMemBucket = ppInfo->pMemBucket; @@ -2442,7 +2485,7 @@ static void buildHistogramInfo(SAPercentileInfo* pInfo) { } static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo* pInfo = NULL; if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { @@ -2455,7 +2498,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { return pInfo; } -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2470,7 +2513,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); assert(pInfo->pHisto->elems != NULL); @@ -2524,7 +2567,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { tHistogramDestroy(&pRes); } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); } @@ -2532,7 +2575,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == MERGE_STAGE) { @@ -2565,7 +2608,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2596,7 +2639,7 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo } static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; @@ -2683,7 +2726,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { @@ -2793,7 +2836,7 @@ enum { INITIAL_VALUE_NOT_ASSIGNED = 0, }; -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2803,7 +2846,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResIn return false; } -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) { +static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2819,7 +2862,7 @@ static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResu } static void deriv_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); void *data = GET_INPUT_DATA_LIST(pCtx); @@ -3179,7 +3222,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { } char *getArithColumnData(void *param, const char* name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *)param; + SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; int32_t index = -1; for (int32_t i = 0; i < pSupport->numOfCols; ++i) { @@ -3195,9 +3238,12 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - -// evaluateExprNodeTree(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); + SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; + + SScalarFuncParam output = {0}; + output.data = pCtx->pOutput; + + evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ @@ -3218,7 +3264,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3238,15 +3284,15 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pRes } static void spread_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t numOfElems = 0; // todo : opt with pre-calculated result // column missing cause the hasNull to be true - if (pCtx->isSmaSet) { - numOfElems = pCtx->size - pCtx->sma.numOfNull; + if (pCtx->isAggSet) { + numOfElems = pCtx->size - pCtx->agg.numOfNull; // all data are null in current data block, ignore current data block if (numOfElems == 0) { @@ -3255,20 +3301,20 @@ static void spread_function(SQLFunctionCtx *pCtx) { if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->sma.min) { - pInfo->min = (double)pCtx->sma.min; + if (pInfo->min > pCtx->agg.min) { + pInfo->min = (double)pCtx->agg.min; } - if (pInfo->max < pCtx->sma.max) { - pInfo->max = (double)pCtx->sma.max; + if (pInfo->max < pCtx->agg.max) { + pInfo->max = (double)pCtx->agg.max; } } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->sma.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->sma.min)); + if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->agg.min))) { + pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->agg.min)); } - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->sma.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->sma.max)); + if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->agg.max))) { + pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->agg.max)); } } @@ -3344,7 +3390,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary */ - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -3377,7 +3423,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * param[2]: end time * @param pCtx */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3400,7 +3446,7 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); TSKEY *tsList = GET_TS_LIST(pCtx); @@ -3642,7 +3688,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si static void twa_function(SQLFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // skip null value @@ -3675,14 +3721,14 @@ static void twa_function(SQLFunctionCtx *pCtx) { */ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->hasResult != DATA_SET_FLAG) { @@ -3872,7 +3918,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; // not initialized since it has been initialized } @@ -3884,7 +3930,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pRe } static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; const char *input = GET_INPUT_DATA_LIST(pCtx); @@ -3904,7 +3950,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf * pTSbuf = pInfo->pTSBuf; @@ -3960,7 +4006,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; } -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) { +static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3978,7 +4024,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResIn } static void rate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4033,13 +4079,13 @@ static void rate_function(SQLFunctionCtx *pCtx) { static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; } static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pRateInfo->hasResult != DATA_SET_FLAG) { @@ -4057,7 +4103,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { } static void irate_function(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4139,7 +4185,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi } static void blockInfo_func(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); int32_t len = *(int32_t*) pCtx->pInput; @@ -4152,7 +4198,7 @@ static void blockInfo_func(SQLFunctionCtx* pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) { +static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDist* pSrc) { STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); assert(pDist != NULL && pSrc != NULL); @@ -4190,7 +4236,7 @@ void block_func_merge(SQLFunctionCtx* pCtx) { STableBlockDist info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); mergeTableBlockDist(pResInfo, &info); taosArrayDestroy(info.dataBlockInfos); @@ -4293,7 +4339,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { } void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); pDist->rowSize = (uint16_t)pCtx->param[0].i; diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index 2710b07bc1..6970b85638 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -21,9 +21,7 @@ #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" -#include "tname.h" #include "thash.h" -#include "tskiplist.h" #include "texpr.h" #include "tvariant.h" diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index f26231a732..aac4eb1f6d 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "os.h" #include "taosdef.h" @@ -27,7 +28,6 @@ #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) -#define GET_FORWARD_DIRECTION_FACTOR(_ord) (((_ord) == TSDB_ORDER_ASC)? 1:-1) static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { @@ -340,9 +340,9 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { return pFillInfo->numOfRows - pFillInfo->index; } -SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, +struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pCol, void* handle) { + struct SFillColInfo* pCol, void* handle) { if (fillType == TSDB_FILL_NONE) { return NULL; } @@ -522,3 +522,33 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t cap return numOfRes; } + +int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { + return pFillInfo->start; +} + +struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const int64_t* fillVal) { + int32_t offset = 0; + + struct SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = &pExpr[i]; + + pFillCol[i].col.bytes = pExprInfo->base.resSchema.bytes; + pFillCol[i].col.type = (int8_t)pExprInfo->base.resSchema.type; + pFillCol[i].col.offset = offset; + pFillCol[i].col.colId = pExprInfo->base.resSchema.colId; + pFillCol[i].tagIndex = -2; + pFillCol[i].flag = pExprInfo->base.colInfo.flag; // always be the normal column for table query + pFillCol[i].functionId = pExprInfo->pExpr->_node.functionId; + pFillCol[i].fillVal.i = fillVal[i]; + + offset += pExprInfo->base.resSchema.bytes; + } + + return pFillCol; +} \ No newline at end of file diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index b3ffb19d7b..8c9d670e87 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -132,9 +132,9 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa void* outputBuf = pOutput->data; if (isStringOp(pExprs->_node.optr)) { outputBuf = realloc(pOutput->data, (left.bytes + right.bytes) * left.num); - OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); } + OperatorFn(&left, &right, outputBuf, TSDB_ORDER_ASC); // Set the result info setScalarFuncParam(pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double), outputBuf, numOfRows); } else if (pExprs->nodeType == TEXPR_UNARYEXPR_NODE) { @@ -174,3 +174,44 @@ SScalarFunctionInfo scalarFunc[1] = { }, }; + +void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { + sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; + sas->pExprInfo = pExprInfo; + if (sas->colList != NULL) { + return; + } + + sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + for(int32_t i = 0; i < sas->numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); + sas->colList[i] = pColData->info; + } + + sas->data = calloc(sas->numOfCols, POINTER_BYTES); + + // set the input column data + for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f); + sas->data[f] = pColumnInfoData->pData; + } +} + +SScalarFunctionSupport* createScalarFuncSupport(int32_t num) { + SScalarFunctionSupport* pSupp = calloc(num, sizeof(SScalarFunctionSupport)); + return pSupp; +} + +void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num) { + if (pSupport == NULL) { + return; + } + + for(int32_t i = 0; i < num; ++i) { + SScalarFunctionSupport* pSupp = &pSupport[i]; + tfree(pSupp->data); + tfree(pSupp->colList); + } + + tfree(pSupport); +} \ No newline at end of file diff --git a/src/query/src/qScript.c b/source/libs/function/src/tscript.c similarity index 99% rename from src/query/src/qScript.c rename to source/libs/function/src/tscript.c index c43b0b3435..7c07b0b783 100644 --- a/src/query/src/qScript.c +++ b/source/libs/function/src/tscript.c @@ -14,12 +14,12 @@ */ #include "os.h" -#include "qScript.h" -#include "ttype.h" +#include "tscript.h" +#include "ttypes.h" #include "tstrbuild.h" -#include "queryLog.h" +//#include "queryLog.h" #include "ttokendef.h" - +#if 0 static ScriptEnvPool *pool = NULL; static ScriptEnv* getScriptEnvFromPool(); @@ -444,3 +444,4 @@ bool isValidScript(char *script, int32_t len) { return ret; } +#endif diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c new file mode 100644 index 0000000000..5815edd701 --- /dev/null +++ b/source/libs/function/src/tudf.c @@ -0,0 +1,124 @@ +#include "tudf.h" + +#if 0 +static char* getUdfFuncName(char* funcname, char* name, int type) { + switch (type) { + case TSDB_UDF_FUNC_NORMAL: + strcpy(funcname, name); + break; + case TSDB_UDF_FUNC_INIT: + sprintf(funcname, "%s_init", name); + break; + case TSDB_UDF_FUNC_FINALIZE: + sprintf(funcname, "%s_finalize", name); + break; + case TSDB_UDF_FUNC_MERGE: + sprintf(funcname, "%s_merge", name); + break; + case TSDB_UDF_FUNC_DESTROY: + sprintf(funcname, "%s_destroy", name); + break; + default: + assert(0); + break; + } + + return funcname; +} + +int32_t initUdfInfo(SUdfInfo* pUdfInfo) { + if (pUdfInfo == NULL) { + return TSDB_CODE_SUCCESS; + } + ////qError("script len: %d", pUdfInfo->contLen); + if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) { + pUdfInfo->isScript = 1; + pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes); + if (pUdfInfo->pScriptCtx == NULL) { + return TSDB_CODE_QRY_SYS_ERROR; + } + tfree(pUdfInfo->content); + + pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit; + if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL + || (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_QRY_SYS_ERROR; + } + + pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; + } + pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy; + + } else { + char path[PATH_MAX] = {0}; + taosGetTmpfilePath("script", path); + + FILE* file = fopen(path, "w+"); + + // TODO check for failure of flush to disk + /*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file); + fclose(file); + tfree(pUdfInfo->content); + + pUdfInfo->path = strdup(path); + + pUdfInfo->handle = taosLoadDll(path); + + if (NULL == pUdfInfo->handle) { + return TSDB_CODE_QRY_SYS_ERROR; + } + + char funcname[TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; + pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); + if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { + return TSDB_CODE_QRY_SYS_ERROR; + } + + pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); + pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); + } + + pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_DESTROY)); + + if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) { + return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init); + } + } + + return TSDB_CODE_SUCCESS; +} + +void destroyUdfInfo(SUdfInfo* pUdfInfo) { + if (pUdfInfo == NULL) { + return; + } + + if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) { + if (pUdfInfo->isScript) { + (*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx); + tfree(pUdfInfo->content); + }else{ + (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init); + } + } + + tfree(pUdfInfo->name); + + if (pUdfInfo->path) { + unlink(pUdfInfo->path); + } + + tfree(pUdfInfo->path); + tfree(pUdfInfo->content); + taosCloseDll(pUdfInfo->handle); + tfree(pUdfInfo); +} + +#endif \ No newline at end of file diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index f37e84927b..922b9c2d44 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -47,13 +47,10 @@ int32_t parserValidateIdToken(SToken* pToken); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr); -int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num); STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo); void columnListCopyAll(SArray* dst, const SArray* src); -void columnListDestroy(SArray* pColumnList); - SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid); @@ -61,6 +58,7 @@ void cleanupTagCond(STagCond* pTagCond); void cleanupColumnCond(SArray** pCond); uint32_t convertRelationalOperator(SToken *pToken); +int32_t getExprFunctionId(SExprInfo *pExprInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index d75637e3c5..68fe08db47 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -39,7 +39,6 @@ int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); -int32_t getExprFunctionId(SExprInfo *pExprInfo); void cleanupFieldInfo(SFieldInfo* pFieldInfo); STableComInfo getTableInfo(const STableMeta* pTableMeta); diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index bbe94e7c78..7dc5fd00b2 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1,6 +1,5 @@ #include "taosmsg.h" #include "parser.h" -#include "parserUtil.h" #include "taoserror.h" #include "tutil.h" #include "ttypes.h" @@ -1482,82 +1481,6 @@ int32_t getNumOfOutput(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } -// todo move to planner module -int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num) { -// if (!pQueryInfo->arithmeticOnAgg) { -// return TSDB_CODE_SUCCESS; -// } -#if 0 - *num = getNumOfOutput(pQueryInfo); - *pExpr = calloc(*(num), POINTER_BYTES); - if ((*pExpr) == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < (*num); ++i) { - SInternalField* pField = getInternalFieldInfo(&pQueryInfo->fieldsInfo, i); - SExprInfo* pSource = pField->pExpr; - - SExprInfo* px = calloc(1, sizeof(SExprInfo)); - (*pExpr)[i] = px; - - SSqlExpr *pse = &px->base; - pse->uid = pTableMetaInfo->pTableMeta->uid; - memcpy(&pse->resSchema, &pSource->base.resSchema, sizeof(SSchema)); - - if (pSource->base.functionId != FUNCTION_ARITHM) { // this should be switched to projection query - pse->numOfParams = 0; // no params for projection query - pse->functionId = FUNCTION_PRJ; - pse->colInfo.colId = pSource->base.resSchema.colId; - - int32_t numOfOutput = (int32_t) taosArrayGetSize(pQueryInfo->exprList); - for (int32_t j = 0; j < numOfOutput; ++j) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, j); - if (p->base.resSchema.colId == pse->colInfo.colId) { - pse->colInfo.colIndex = j; - break; - } - } - - pse->colInfo.flag = TSDB_COL_NORMAL; - strncpy(pse->colInfo.name, pSource->base.resSchema.name, tListLen(pse->colInfo.name)); - - // TODO restore refactor - int32_t functionId = pSource->base.functionId; - if (pSource->base.functionId == FUNCTION_FIRST_DST) { - functionId = FUNCTION_FIRST; - } else if (pSource->base.functionId == FUNCTION_LAST_DST) { - functionId = FUNCTION_LAST; - } else if (pSource->base.functionId == FUNCTION_STDDEV_DST) { - functionId = FUNCTION_STDDEV; - } - - int32_t inter = 0; - getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resSchema.type, - &pse->resSchema.bytes, &inter, 0, false/*, NULL*/); - pse->colType = pse->resSchema.type; - pse->colBytes = pse->resSchema.bytes; - - } else { // arithmetic expression - pse->colInfo.colId = pSource->base.colInfo.colId; - pse->colType = pSource->base.colType; - pse->colBytes = pSource->base.colBytes; - pse->resSchema.bytes = sizeof(double); - pse->resSchema.type = TSDB_DATA_TYPE_DOUBLE; - - pse->functionId = pSource->base.functionId; - pse->numOfParams = pSource->base.numOfParams; - - for (int32_t j = 0; j < pSource->base.numOfParams; ++j) { - taosVariantAssign(&pse->param[j], &pSource->base.param[j]); -// buildArithmeticExprFromMsg(px, NULL); - } - } - } -#endif - return TSDB_CODE_SUCCESS; -} - int32_t getColFilterSerializeLen(SQueryStmtInfo* pQueryInfo) { int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList); int32_t len = 0; diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index e9d4e96337..27a96b539e 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -41,7 +41,7 @@ typedef struct SQueryPlanNode { SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations - int32_t numOfOutput; // number of result columns, which is also the number of pExprs + int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information // previous operator to generated result for current node to process // in case of join, multiple prev nodes exist. @@ -50,6 +50,7 @@ typedef struct SQueryPlanNode { } SQueryPlanNode; typedef struct SQueryDistPlanNode { + SQueryNodeBasicInfo info; } SQueryDistPlanNode; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 1a6c496cd0..969b2c7622 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -104,7 +104,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla pNode->tableInfo.tableName = strdup(pTableInfo->tableName); } - pNode->numOfOutput = numOfOutput; + pNode->numOfExpr = numOfOutput; pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); for(int32_t i = 0; i < numOfOutput; ++i) { @@ -234,8 +234,8 @@ static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryIn if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput); + pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); + memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo); } @@ -375,14 +375,14 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len1 = sprintf(buf + len, "cols: "); len += len1; - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* p = &pExprInfo->base; len1 = sprintf(buf + len, "[%s #%d]", p->resSchema.name, p->resSchema.colId); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len, ", "); len += len1; } @@ -398,12 +398,12 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_AGGREGATE: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len, ", "); len += len1; } @@ -415,12 +415,12 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_TIMEWINDOW: { - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } @@ -441,14 +441,14 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } case QNODE_GROUPBY: { // todo hide the invisible column - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSqlExpr* pExpr = &pExprInfo->base; len1 = sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } @@ -473,11 +473,11 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len += len1; // todo get the correct fill data type - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } @@ -501,14 +501,14 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, len1 = sprintf(buf + len,"cols: "); len += len1; - for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); SSchema* resSchema = &pExprInfo->base.resSchema; len1 = sprintf(buf + len,"[%s #%d]", resSchema->name, resSchema->colId); len += len1; - if (i < pQueryNode->numOfOutput - 1) { + if (i < pQueryNode->numOfExpr - 1) { len1 = sprintf(buf + len,", "); len += len1; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5b7ab5db65..22166245f6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -23,9 +23,7 @@ #include #include "../../../include/client/taos.h" #include "os.h" -#include "qFilter.h" #include "qPlan.h" -#include "qScript.h" #include "qSqlparser.h" #include "qTableMeta.h" #include "qUtil.h" @@ -33,10 +31,12 @@ #include "taosmsg.h" #include "tcompare.h" #include "texpr.h" +#include "tfilter.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" #include "tsclient.h" +#include "tscript.h" #include "tstrbuild.h" #include "ttoken.h" #include "ttokendef.h" diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 8af340030c..0ca8e49fa7 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -15,17 +15,17 @@ #include "os.h" #include "taosmsg.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" #include "tref.h" #include "trpc.h" -#include "tnote.h" -#include "ttimer.h" -#include "tsched.h" #include "tscLog.h" +#include "tsched.h" #include "tsclient.h" -#include "tglobal.h" -#include "tconfig.h" +#include "tscript.h" +#include "ttimer.h" #include "ttimezone.h" -#include "qScript.h" // global, not configurable #define TSC_VAR_NOT_RELEASE 1 diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index d4116fbfb2..db165aa206 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -258,7 +258,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha (_r)->initialized = false; \ } while (0) -static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) { +static FORCE_INLINE void initResultRowEntry(SResultRowCellInfo *pResInfo, int32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 746c5f8569..5c18c5aef6 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -1,9 +1,9 @@ #ifndef TDENGINE_QTABLEUTIL_H #define TDENGINE_QTABLEUTIL_H -#include "tsdb.h" //todo tsdb should not be here #include "qSqlparser.h" -#include "qFilter.h" +#include "tfilter.h" +#include "tsdb.h" //todo tsdb should not be here typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index c0c6d7a140..037b60343e 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -425,7 +425,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo } memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResultInfo, pCtx->interBufBytes); + initResultRowEntry(pResultInfo, pCtx->interBufBytes); return true; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ec576eec8b..4627377a8e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -21,7 +21,6 @@ #include "hash.h" #include "qExecutor.h" #include "qResultbuf.h" -#include "qScript.h" #include "qUtil.h" #include "queryLog.h" #include "tcompare.h" @@ -29,6 +28,7 @@ #include "texpr.h" #include "tlosertree.h" #include "tscLog.h" +#include "tscript.h" #include "ttype.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 5e8ff126d1..a9bfe90d32 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -12,11 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "hash.h" #include "os.h" #include "queryLog.h" -#include "qFilter.h" #include "tcompare.h" -#include "hash.h" +#include "tfilter.h" #include "tscUtil.h" OptrStr gOptrStr[] = { From 89e324c89926cdc8e842f20cafb2a3cb0de30fd2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 Nov 2021 15:08:00 +0800 Subject: [PATCH 04/26] [td-10564]fix compiler error. --- include/libs/function/function.h | 48 ++++-- include/util/tdef.h | 4 - source/libs/executor/src/executorimpl.c | 153 ++---------------- source/libs/function/src/taggfunction.c | 202 ++++++++++++------------ source/libs/function/src/tfunction.c | 24 +++ source/libs/function/src/tudf.c | 81 +++++++++- 6 files changed, 245 insertions(+), 267 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index b91cc83255..3f5422824a 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -26,8 +26,8 @@ extern "C" { #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results -#define FUNCTION_SCALAR 1 -#define FUNCTION_AGG 2 +#define FUNCTION_TYPE_SCALAR 1 +#define FUNCTION_TYPE_AGG 2 #define TOP_BOTTOM_QUERY_LIMIT 100 #define FUNCTIONS_NAME_MAX_LENGTH 16 @@ -108,8 +108,23 @@ typedef struct SExtTagsInfo { struct SQLFunctionCtx **pTagCtxList; } SExtTagsInfo; +typedef struct SResultDataInfo { + int16_t type; + int16_t bytes; + int32_t intermediateBytes; +} SResultDataInfo; + #define GET_RES_INFO(ctx) ((ctx)->resultInfo) +typedef struct SFunctionFpSet { + bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(struct SQLFunctionCtx *pCtx); + + // finalizer must be called after all exec has been executed to generated final result. + void (*finalize)(struct SQLFunctionCtx *pCtx); + void (*combine)(struct SQLFunctionCtx *pCtx); +} SFunctionFpSet; + // sql function runtime context typedef struct SQLFunctionCtx { int32_t size; // number of rows @@ -118,9 +133,7 @@ typedef struct SQLFunctionCtx { int16_t inputType; int16_t inputBytes; - int16_t outputType; - int16_t outputBytes; // size of results, determined by function and input column data type - int32_t interBufBytes; // internal buffer size + SResultDataInfo resDataInfo; bool hasNull; // null value exist in current block bool requireNull; // require null in some function bool stableQuery; @@ -135,11 +148,13 @@ typedef struct SQLFunctionCtx { SVariant tag; bool isAggSet; - SColumnDataAgg agg; + SColumnDataAgg agg; struct SResultRowEntryInfo *resultInfo; SExtTagsInfo tagInfo; SPoint1 start; SPoint1 end; + + SFunctionFpSet* fpSet; } SQLFunctionCtx; enum { @@ -179,11 +194,11 @@ typedef struct SAggFunctionInfo { uint16_t status; bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*exec)(SQLFunctionCtx *pCtx); + void (*addInput)(SQLFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*xFinalize)(SQLFunctionCtx *pCtx); - void (*mergeFunc)(SQLFunctionCtx *pCtx); + void (*finalize)(SQLFunctionCtx *pCtx); + void (*combine)(SQLFunctionCtx *pCtx); int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; @@ -194,15 +209,9 @@ typedef struct SScalarFunctionInfo { uint8_t functionId; // index of scalar function bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*exec)(SQLFunctionCtx *pCtx); + void (*addInput)(SQLFunctionCtx *pCtx); } SScalarFunctionInfo; -typedef struct SResultDataInfo { - int16_t type; - int16_t bytes; - int32_t intermediateBytes; -} SResultDataInfo; - typedef struct SMultiFunctionsDesc { bool stableQuery; bool groupbyColumn; @@ -280,6 +289,13 @@ int64_t getFillInfoStart(struct SFillInfo *pFillInfo); int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// udf api +struct SUdfInfo; + +void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); +void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index fca9a1395b..c7a421a55f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -318,10 +318,6 @@ do { \ #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode -#define TSDB_UDF_TYPE_SCALAR 1 -#define TSDB_UDF_TYPE_AGGREGATE 2 - - /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 70b3c8982f..9b3cbcf9cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -24,7 +24,6 @@ #include "function.h" #include "tcompare.h" #include "tcompression.h" -#include "tlosertree.h" #include "ttypes.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -356,39 +355,6 @@ void* destroyOutputBuf(SSDataBlock* pBlock) { return NULL; } -//int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { -// SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// bool hasMainFunction = hasMainOutput(pQueryAttr); -// -// int32_t maxOutput = 0; -// for (int32_t j = 0; j < numOfOutput; ++j) { -// int32_t id = pCtx[j].functionId; -// -// /* -// * ts, tag, tagprj function can not decide the output number of current query -// * the number of output result is decided by main output -// */ -// if (hasMainFunction && (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ)) { -// continue; -// } -// -// SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); -// if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { -// maxOutput = pResInfo->numOfRes; -// } -// } -// -// assert(maxOutput >= 0); -// return maxOutput; -//} -// -//static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { -// for (int32_t j = 0; j < numOfOutput; ++j) { -// SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); -// pResInfo->numOfRes = 0; -// } -//} - static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return true; // bool hasTags = false; @@ -848,8 +814,6 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, pResultRowInfo->curPos = i + 1; // current not closed result object } } - - //pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey; } static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQueryAttr* pQueryAttr, TSKEY lastKey) { @@ -903,80 +867,6 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc return num; } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { -#if 0 - int32_t output = 0; - - if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { - //qError("empty udf function, type:%d", type); - return; - } - -// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); - - switch (type) { - case TSDB_UDF_FUNC_NORMAL: - if (pUdfInfo->isScript) { - (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, - (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, - (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes); - } else { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - - (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, - pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init); - } - - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { - pCtx->resultInfo->numOfRes = output; - } else { - pCtx->resultInfo->numOfRes += output; - } - - if (pCtx->resultInfo->numOfRes > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_MERGE: - if (pUdfInfo->isScript) { - (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); - } else { - (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); - } - - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - - case TSDB_UDF_FUNC_FINALIZE: { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); - if (pUdfInfo->isScript) { - (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); - } else { - (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); - } - // set the output value exist - pCtx->resultInfo->numOfRes = output; - if (output > 0) { - pCtx->resultInfo->hasResult = DATA_SET_FLAG; - } - - break; - } - } -#endif - -} - static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1004,14 +894,8 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx pCtx[k].isAggSet = false; } - int32_t functionId = pCtx[k].functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { -// if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo -// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; -// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); -// } else { -// aAggs[functionId].xFunction(&pCtx[k]); -// } + pCtx[k].fpSet->addInput(&pCtx[k]); } // restore it @@ -1020,9 +904,8 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx } } - -static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, - TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { +static int32_t getNextQualifiedWindow(SQueryAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { getNextTimeWindow(pQueryAttr, pNext); // next time window is not in current block @@ -1244,14 +1127,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct - - int32_t functionId = pCtx[k].functionId; -// if (functionId < 0) { -// SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; -// doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL); -// } else { -// aAggs[functionId].xFunction(&pCtx[k]); -// } + pCtx[k].fpSet->addInput(&pCtx[k]); } } } @@ -1940,7 +1816,7 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { int32_t functionId = pCtx[i].functionId; if (functionId == FUNCTION_TAG_DUMMY || functionId == FUNCTION_TS_DUMMY) { - tagLen += pCtx[i].outputBytes; + tagLen += pCtx[i].resDataInfo.bytes; pTagCtx[num++] = &pCtx[i]; } else if (1/*(aAggs[functionId].status & FUNCSTATE_SELECTIVITY) != 0*/) { p = &pCtx[i]; @@ -1996,13 +1872,13 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr pCtx->ptsOutputBuf = NULL; - pCtx->outputBytes = pSqlExpr->resSchema.bytes; - pCtx->outputType = pSqlExpr->resSchema.type; + pCtx->resDataInfo.bytes = pSqlExpr->resSchema.bytes; + pCtx->resDataInfo.type = pSqlExpr->resSchema.type; pCtx->order = pQueryAttr->order.order; // pCtx->functionId = pSqlExpr->functionId; pCtx->stableQuery = pQueryAttr->stableQuery; - pCtx->interBufBytes = pSqlExpr->interBytes; + pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes; pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; @@ -3577,11 +3453,7 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { continue; } -// if (pCtx[j].functionId < 0) { // todo udf initialization -// continue; -// } else { -// aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo); -// } + pCtx[j].fpSet->init(&pCtx[j], pCtx[j].resultInfo); } } @@ -3737,12 +3609,12 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - offset += pCtx[i].outputBytes; + offset += pCtx[i].resDataInfo.bytes; continue; } pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, bufPage, pResult->offset, offset); - offset += pCtx[i].outputBytes; + offset += pCtx[i].resDataInfo.bytes; int32_t functionId = pCtx[i].functionId; if (functionId < 0) { @@ -3807,7 +3679,7 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF int16_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResult->offset, offset); - offset += pCtx[i].outputBytes; + offset += pCtx[i].resDataInfo.bytes; int32_t functionId = pCtx[i].functionId; if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { @@ -6881,7 +6753,6 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { } SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; STagScanInfo *pInfo = pOperator->info; diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index f4e48f2faa..a2e6c3d3ac 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -40,7 +40,7 @@ #define GET_TRUE_DATA_TYPE() \ int32_t type = 0; \ if (pCtx->currentStage == MERGE_STAGE) { \ - type = pCtx->outputType; \ + type = pCtx->resDataInfo.type; \ assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ } else { \ type = pCtx->inputType; \ @@ -61,10 +61,10 @@ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ + __ctx->tag.i = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ } \ - aggFunc[FUNCTION_TAG].exec(__ctx); \ + aggFunc[FUNCTION_TAG].addInput(__ctx); \ } \ } while (0) @@ -72,7 +72,7 @@ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - aggFunc[FUNCTION_TAG].exec(__ctx); \ + aggFunc[FUNCTION_TAG].addInput(__ctx); \ } \ } while (0); @@ -478,8 +478,8 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf return false; } - memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); - initResultRowEntry(pResultInfo, pCtx->interBufBytes); + memset(pCtx->pOutput, 0, (size_t)pCtx->resDataInfo.bytes); + initResultRowEntry(pResultInfo, pCtx->resDataInfo.intermediateBytes); return true; } @@ -493,7 +493,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf static void function_finalizer(SQLFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } doFinalizer(pCtx); @@ -914,7 +914,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } @@ -924,7 +924,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } @@ -1000,7 +1000,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; } - aggFunc[FUNCTION_TAG].exec(__ctx); + aggFunc[FUNCTION_TAG].addInput(__ctx); } } } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { @@ -1248,7 +1248,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; - aggFunc[FUNCTION_TAG].exec(__ctx); + aggFunc[FUNCTION_TAG].addInput(__ctx); } notNullElems++; @@ -1304,7 +1304,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 1); + int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1315,7 +1315,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { } static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 0); + int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1423,7 +1423,7 @@ static void stddev_finalizer(SQLFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else { double *retValue = (double *)pCtx->pOutput; SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); @@ -1557,7 +1557,7 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else { double *retValue = (double *)pCtx->pOutput; SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); @@ -1665,16 +1665,16 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { assert(pCtx->stableQuery); char * pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } // The param[1] is used to keep the initial value of max ts value - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i > pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->outputBytes); + if (pCtx->param[1].nType != pCtx->resDataInfo.type || pCtx->param[1].i > pInput->ts) { + memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); pCtx->param[1].i = pInput->ts; - pCtx->param[1].nType = pCtx->outputType; + pCtx->param[1].nType = pCtx->resDataInfo.type; DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } @@ -1799,7 +1799,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } @@ -1808,10 +1808,10 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { * param[1] used to keep the corresponding timestamp to decide if current result is * the true last result */ - if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i < pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->outputBytes); + if (pCtx->param[1].nType != pCtx->resDataInfo.type || pCtx->param[1].i < pInput->ts) { + memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); pCtx->param[1].i = pInput->ts; - pCtx->param[1].nType = pCtx->outputType; + pCtx->param[1].nType = pCtx->resDataInfo.type; DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } @@ -1853,7 +1853,7 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } @@ -1880,7 +1880,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 } taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); - size += pTagInfo->pTagCtxList[i]->outputBytes; + size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; } } } @@ -2101,9 +2101,9 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { for (int32_t i = 0; i < len; ++i, output += step) { int16_t offset = 0; for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->outputBytes); - offset += pCtx->tagInfo.pTagCtxList[j]->outputBytes; - pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes; + memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); + offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; + pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; } } @@ -2262,7 +2262,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType; + int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2319,7 +2319,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->outputType; + int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2469,7 +2469,7 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { tMemBucket * pMemBucket = ppInfo->pMemBucket; if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null assert(ppInfo->numOfElems == 0); - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else { SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); } @@ -2588,7 +2588,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, res, sizeof(double)); free(res); } else { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } } else { @@ -2599,7 +2599,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, res, sizeof(double)); free(res); } else { // no need to free - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } } @@ -2730,7 +2730,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } @@ -2794,16 +2794,16 @@ static void col_project_function(SQLFunctionCtx *pCtx) { static void tag_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); - assert(pCtx->inputBytes == pCtx->outputBytes); + assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); + taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); char* data = pCtx->pOutput; - pCtx->pOutput += pCtx->outputBytes; + pCtx->pOutput += pCtx->resDataInfo.bytes; // directly copy from the first one for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->pOutput, data, pCtx->outputBytes); - pCtx->pOutput += pCtx->outputBytes; + memmove(pCtx->pOutput, data, pCtx->resDataInfo.bytes); + pCtx->pOutput += pCtx->resDataInfo.bytes; } } @@ -2821,7 +2821,7 @@ static void tag_function(SQLFunctionCtx *pCtx) { if (pCtx->currentStage == MERGE_STAGE) { copy_function(pCtx); } else { - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); + taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); } } @@ -3396,7 +3396,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } @@ -3406,7 +3406,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); return; } @@ -3763,7 +3763,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { *(TSKEY *)pCtx->pOutput = pCtx->startTs; } else if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else if (type == TSDB_FILL_SET_VALUE) { taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); } else { @@ -3772,13 +3772,13 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); } } else if (type == TSDB_FILL_NEXT) { if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); } } else if (type == TSDB_FILL_LINEAR) { SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; @@ -3790,7 +3790,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); } } else { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); @@ -3817,7 +3817,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { skey = ekey; } } - assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); } else if (type == TSDB_FILL_NEXT) { TSKEY ekey = skey; char* val = NULL; @@ -3837,7 +3837,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { val = (char*)pCtx->pInput; } - assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, val, pCtx->resDataInfo.bytes, pCtx->inputType); } else if (type == TSDB_FILL_LINEAR) { if (pCtx->size <= 1) { return; @@ -3863,7 +3863,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { if (isNull(start, srcType) || isNull(end, srcType)) { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType); + taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, srcType); } } else { setNull(pCtx->pOutput, srcType, pCtx->inputBytes); @@ -4382,7 +4382,7 @@ int32_t functionCompatList[] = { SAggFunctionInfo aggFunc[34] = {{ // 0, count function does not invoke the finalize function "count", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_COUNT, FUNCTION_COUNT, BASIC_FUNC_SO, @@ -4395,7 +4395,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 1 "sum", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_SUM, FUNCTION_SUM, BASIC_FUNC_SO, @@ -4408,7 +4408,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 2 "avg", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_AVG, FUNCTION_AVG, BASIC_FUNC_SO, @@ -4421,7 +4421,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 3 "min", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_MIN, FUNCTION_MIN, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, @@ -4434,8 +4434,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 4 "max", - FUNCTION_AGG, - FUNCTION_MAX, + FUNCTION_TYPE_AGG, + FUNCTION_MAX, FUNCTION_MAX, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, max_func_setup, @@ -4447,8 +4447,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 5 "stddev", - FUNCTION_AGG, - FUNCTION_STDDEV, + FUNCTION_TYPE_AGG, + FUNCTION_STDDEV, FUNCTION_STDDEV_DST, FUNCSTATE_SO | FUNCSTATE_STREAM, function_setup, @@ -4460,8 +4460,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 6 "percentile", - FUNCTION_AGG, - FUNCTION_PERCT, + FUNCTION_TYPE_AGG, + FUNCTION_PERCT, FUNCTION_INVALID_ID, FUNCSTATE_SO | FUNCSTATE_STREAM, percentile_function_setup, @@ -4473,8 +4473,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 7 "apercentile", - FUNCTION_AGG, - FUNCTION_APERCT, + FUNCTION_TYPE_AGG, + FUNCTION_APERCT, FUNCTION_APERCT, FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE, apercentile_function_setup, @@ -4486,8 +4486,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 8 "first", - FUNCTION_AGG, - FUNCTION_FIRST, + FUNCTION_TYPE_AGG, + FUNCTION_FIRST, FUNCTION_FIRST_DST, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, function_setup, @@ -4499,8 +4499,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 9 "last", - FUNCTION_AGG, - FUNCTION_LAST, + FUNCTION_TYPE_AGG, + FUNCTION_LAST, FUNCTION_LAST_DST, BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY, function_setup, @@ -4512,8 +4512,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 10 "last_row", - FUNCTION_AGG, - FUNCTION_LAST_ROW, + FUNCTION_TYPE_AGG, + FUNCTION_LAST_ROW, FUNCTION_LAST_ROW, FUNCSTATE_SO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, first_last_function_setup, @@ -4525,8 +4525,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 11 "top", - FUNCTION_AGG, - FUNCTION_TOP, + FUNCTION_TYPE_AGG, + FUNCTION_TOP, FUNCTION_TOP, FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, top_bottom_function_setup, @@ -4538,8 +4538,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 12 "bottom", - FUNCTION_AGG, - FUNCTION_BOTTOM, + FUNCTION_TYPE_AGG, + FUNCTION_BOTTOM, FUNCTION_BOTTOM, FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, top_bottom_function_setup, @@ -4551,8 +4551,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 13 "spread", - FUNCTION_AGG, - FUNCTION_SPREAD, + FUNCTION_TYPE_AGG, + FUNCTION_SPREAD, FUNCTION_SPREAD, BASIC_FUNC_SO, spread_function_setup, @@ -4564,8 +4564,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 14 "twa", - FUNCTION_AGG, - FUNCTION_TWA, + FUNCTION_TYPE_AGG, + FUNCTION_TWA, FUNCTION_TWA, BASIC_FUNC_SO | FUNCSTATE_NEED_TS, twa_function_setup, @@ -4577,8 +4577,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 15 "leastsquares", - FUNCTION_AGG, - FUNCTION_LEASTSQR, + FUNCTION_TYPE_AGG, + FUNCTION_LEASTSQR, FUNCTION_INVALID_ID, FUNCSTATE_SO | FUNCSTATE_STREAM, leastsquares_function_setup, @@ -4590,8 +4590,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 16 "ts", - FUNCTION_AGG, - FUNCTION_TS, + FUNCTION_TYPE_AGG, + FUNCTION_TS, FUNCTION_TS, BASIC_FUNC_SO | FUNCSTATE_NEED_TS, function_setup, @@ -4603,7 +4603,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 17 "ts", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_TS_DUMMY, FUNCTION_TS_DUMMY, BASIC_FUNC_SO | FUNCSTATE_NEED_TS, @@ -4616,7 +4616,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 18 "tag_dummy", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_TAG_DUMMY, FUNCTION_TAG_DUMMY, BASIC_FUNC_SO, @@ -4629,7 +4629,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 19 "ts", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_TS_COMP, FUNCTION_TS_COMP, FUNCSTATE_MO | FUNCSTATE_NEED_TS, @@ -4642,7 +4642,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 20 "tag", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_TAG, FUNCTION_TAG, BASIC_FUNC_SO, @@ -4655,7 +4655,7 @@ SAggFunctionInfo aggFunc[34] = {{ {//TODO this is a scala function // 21, column project sql function "colprj", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_PRJ, FUNCTION_PRJ, BASIC_FUNC_MO | FUNCSTATE_NEED_TS, @@ -4668,7 +4668,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 22, multi-output, tag function has only one result "tagprj", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_TAGPRJ, FUNCTION_TAGPRJ, BASIC_FUNC_MO, @@ -4681,7 +4681,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 23 "arithmetic", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_ARITHM, FUNCTION_ARITHM, FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS, @@ -4694,7 +4694,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 24 "diff", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_DIFF, FUNCTION_INVALID_ID, FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, @@ -4708,8 +4708,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 25 "first_dist", - FUNCTION_AGG, - FUNCTION_FIRST_DST, + FUNCTION_TYPE_AGG, + FUNCTION_FIRST_DST, FUNCTION_FIRST_DST, BASIC_FUNC_SO | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, first_last_function_setup, @@ -4721,7 +4721,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 26 "last_dist", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_LAST_DST, FUNCTION_LAST_DST, BASIC_FUNC_SO | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, @@ -4734,7 +4734,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 27 "stddev", // return table id and the corresponding tags for join match and subscribe - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_STDDEV_DST, FUNCTION_AVG, FUNCSTATE_SO | FUNCSTATE_STABLE, @@ -4747,7 +4747,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 28 "interp", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_INTERP, FUNCTION_INTERP, FUNCSTATE_SO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS , @@ -4760,7 +4760,7 @@ SAggFunctionInfo aggFunc[34] = {{ { // 29 "rate", - FUNCTION_AGG, + FUNCTION_TYPE_AGG, FUNCTION_RATE, FUNCTION_RATE, BASIC_FUNC_SO | FUNCSTATE_NEED_TS, @@ -4773,8 +4773,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 30 "irate", - FUNCTION_AGG, - FUNCTION_IRATE, + FUNCTION_TYPE_AGG, + FUNCTION_IRATE, FUNCTION_IRATE, BASIC_FUNC_SO | FUNCSTATE_NEED_TS, rate_function_setup, @@ -4786,8 +4786,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 31 "tbid", // return table id and the corresponding tags for join match and subscribe - FUNCTION_AGG, - FUNCTION_TID_TAG, + FUNCTION_TYPE_AGG, + FUNCTION_TID_TAG, FUNCTION_TID_TAG, FUNCSTATE_MO | FUNCSTATE_STABLE, function_setup, @@ -4798,8 +4798,8 @@ SAggFunctionInfo aggFunc[34] = {{ }, { //32 "derivative", // return table id and the corresponding tags for join match and subscribe - FUNCTION_AGG, - FUNCTION_DERIVATIVE, + FUNCTION_TYPE_AGG, + FUNCTION_DERIVATIVE, FUNCTION_INVALID_ID, FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY, deriv_function_setup, @@ -4811,8 +4811,8 @@ SAggFunctionInfo aggFunc[34] = {{ { // 33 "_block_dist", // return table id and the corresponding tags for join match and subscribe - FUNCTION_AGG, - FUNCTION_BLKINFO, + FUNCTION_TYPE_AGG, + FUNCTION_BLKINFO, FUNCTION_BLKINFO, FUNCSTATE_SO | FUNCSTATE_STABLE, function_setup, diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 0136bbf493..e9a59f7d69 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -6,6 +6,7 @@ #include "tscalarfunction.h" static SHashObj* functionHashTable = NULL; +static SHashObj* udfHashTable = NULL; static void doInitFunctionHashTable() { int numOfEntries = tListLen(aggFunc); @@ -23,6 +24,8 @@ static void doInitFunctionHashTable() { SScalarFunctionInfo* ptr = &scalarFunc[i]; taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES); } + + udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true); } static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT; @@ -46,6 +49,27 @@ const char* qGetFunctionName(int32_t functionId) { } +SAggFunctionInfo* qGetFunctionInfo(const char* name, int32_t len) { + pthread_once(&functionHashTableInit, doInitFunctionHashTable); + + SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len); + if (pInfo != NULL) { + return (*pInfo); + } else { + return NULL; + } +} + +void qAddUdfInfo(uint64_t id, SUdfInfo* pUdfInfo) { + int32_t len = (uint32_t)strlen(pUdfInfo->name); + taosHashPut(udfHashTable, pUdfInfo->name, len, (void*)&pUdfInfo, POINTER_BYTES); +} + +void qRemoveUdfInfo(uint64_t id, SUdfInfo* pUdfInfo) { + int32_t len = (uint32_t)strlen(pUdfInfo->name); + taosHashRemove(udfHashTable, pUdfInfo->name, len); +} + bool isTagsQuery(SArray* pFunctionIdList) { int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList); for (int32_t i = 0; i < num; ++i) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5815edd701..095c5a6bb0 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1,6 +1,5 @@ #include "tudf.h" -#if 0 static char* getUdfFuncName(char* funcname, char* name, int type) { switch (type) { case TSDB_UDF_FUNC_NORMAL: @@ -26,6 +25,7 @@ static char* getUdfFuncName(char* funcname, char* name, int type) { return funcname; } +#if 0 int32_t initUdfInfo(SUdfInfo* pUdfInfo) { if (pUdfInfo == NULL) { return TSDB_CODE_SUCCESS; @@ -47,7 +47,7 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal; - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize; pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge; } @@ -55,7 +55,7 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { } else { char path[PATH_MAX] = {0}; - taosGetTmpfilePath("script", path); + taosGetTmpfilePath("script", path, tsTempDir); FILE* file = fopen(path, "w+"); @@ -72,7 +72,7 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { return TSDB_CODE_QRY_SYS_ERROR; } - char funcname[TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; + char funcname[FUNCTIONS_NAME_MAX_LENGTH + 10] = {0}; pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_NORMAL)); if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) { return TSDB_CODE_QRY_SYS_ERROR; @@ -80,7 +80,7 @@ int32_t initUdfInfo(SUdfInfo* pUdfInfo) { pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_INIT)); - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + if (pUdfInfo->funcType == FUNCTION_TYPE_AGG) { pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE)); pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(funcname, pUdfInfo->name, TSDB_UDF_FUNC_MERGE)); } @@ -121,4 +121,75 @@ void destroyUdfInfo(SUdfInfo* pUdfInfo) { tfree(pUdfInfo); } +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { + int32_t output = 0; + + if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { + //qError("empty udf function, type:%d", type); + return; + } + +// //qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]); + + switch (type) { + case TSDB_UDF_FUNC_NORMAL: + if (pUdfInfo->isScript) { + (*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx, + (char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput, + (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); + } else { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); + + (*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, + pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes, &pUdfInfo->init); + } + + if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + pCtx->resultInfo->numOfRes = output; + } else { + pCtx->resultInfo->numOfRes += output; + } + + if (pCtx->resultInfo->numOfRes > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + + case TSDB_UDF_FUNC_MERGE: + if (pUdfInfo->isScript) { + (*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output); + } else { + (*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init); + } + + // set the output value exist + pCtx->resultInfo->numOfRes = output; + if (output > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + + case TSDB_UDF_FUNC_FINALIZE: { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo); + if (pUdfInfo->isScript) { + (*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output); + } else { + (*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init); + } + // set the output value exist + pCtx->resultInfo->numOfRes = output; + if (output > 0) { + pCtx->resultInfo->hasResult = DATA_SET_FLAG; + } + + break; + } + } +} + #endif \ No newline at end of file From 55f3ac17016021891c54068e3412df2d5fd5e202 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 Nov 2021 13:24:19 +0800 Subject: [PATCH 05/26] [td-10564] refactor and add test cases. --- include/common/common.h | 12 +- include/libs/function/function.h | 44 +- include/libs/parser/parser.h | 29 - include/util/tdef.h | 14 +- source/libs/executor/src/executorimpl.c | 55 +- source/libs/function/inc/tscalarfunction.h | 2 +- source/libs/function/src/tfill.c | 4 +- source/libs/function/src/tfunction.c | 3 +- source/libs/function/src/tscalarfunction.c | 161 +++++- source/libs/function/src/tunaryoperator.c | 164 +----- source/libs/parser/inc/parserUtil.h | 3 +- source/libs/parser/src/astValidate.c | 516 +++++++++++------- source/libs/parser/src/parser.c | 3 +- source/libs/parser/src/parserUtil.c | 115 ++-- source/libs/parser/src/queryInfoUtil.c | 59 ++- source/libs/parser/test/parserTests.cpp | 586 +++++++++++---------- source/libs/parser/test/plannerTest.cpp | 8 +- source/libs/planner/src/planner.c | 2 +- 18 files changed, 974 insertions(+), 806 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index bcb48d2ce2..8f9b595d7a 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -68,6 +68,13 @@ typedef struct SColumnInfoData { //====================================================================================================================== // the following structure shared by parser and executor +typedef struct SColumn { + uint64_t uid; + char name[TSDB_COL_NAME_LEN]; + int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) + SColumnInfo info; +} SColumn; + typedef struct SLimit { int64_t limit; int64_t offset; @@ -89,8 +96,9 @@ typedef struct SGroupbyExpr { typedef struct SSqlExpr { char token[TSDB_COL_NAME_LEN]; // original token SSchema resSchema; - SColIndex colInfo; // there may be mutiple input columns - uint64_t uid; // table uid, todo refactor use the pointer + + int32_t numOfCols; + SColumn* pColumns; // data columns that are required by query int32_t interBytes; // inter result buffer size int16_t numOfParams; // argument value of each function SVariant param[3]; // parameters are not more than 3 diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 3f5422824a..1bc0128dbe 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -161,6 +161,7 @@ enum { TEXPR_NODE_DUMMY = 0x0, TEXPR_BINARYEXPR_NODE= 0x1, TEXPR_UNARYEXPR_NODE = 0x2, + TEXPR_FUNCTION_NODE = 0x3, TEXPR_COL_NODE = 0x4, TEXPR_VALUE_NODE = 0x8, }; @@ -169,10 +170,7 @@ typedef struct tExprNode { uint8_t nodeType; union { struct { - union { - int32_t optr; // binary operator - int32_t functionId;// unary operator - }; + int32_t optr; // binary operator void *info; // support filter operation on this expression only available for leaf node struct tExprNode *pLeft; // left child pointer struct tExprNode *pRight; // right child pointer @@ -180,18 +178,32 @@ typedef struct tExprNode { SSchema *pSchema;// column node struct SVariant *pVal; // value node + + struct {// function node + char *functionName; + int32_t functionId; + int32_t num; + + // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the + // calculation instead. + // E.g., Cov(col1, col2), the column information, w.r.t. the col1 and col2, is kept in pChild nodes. + // The concat function, concat(col1, col2), is a binary scalar + // operator and is kept in the attribute of _node. + struct tExprNode **pChild; + } _function; }; } tExprNode; +//TODO create? void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); typedef struct SAggFunctionInfo { - char name[FUNCTIONS_NAME_MAX_LENGTH]; - int8_t type; // Scalar function or aggregation function - uint8_t functionId; // Function Id - int8_t sFunctionId; // Transfer function for super table query - uint16_t status; + char name[FUNCTIONS_NAME_MAX_LENGTH]; + int8_t type; // Scalar function or aggregation function + uint32_t functionId; // Function Id + int8_t sFunctionId; // Transfer function for super table query + uint16_t status; bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment void (*addInput)(SQLFunctionCtx *pCtx); @@ -203,13 +215,13 @@ typedef struct SAggFunctionInfo { int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; -typedef struct SScalarFunctionInfo { - char name[FUNCTIONS_NAME_MAX_LENGTH]; - int8_t type; // scalar function or aggregation function - uint8_t functionId; // index of scalar function +struct SScalarFuncParam; - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SQLFunctionCtx *pCtx); +typedef struct SScalarFunctionInfo { + char name[FUNCTIONS_NAME_MAX_LENGTH]; + int8_t type; // scalar function or aggregation function + uint32_t functionId; // index of scalar function + void (*process)(const struct SScalarFuncParam *pInput, struct SScalarFuncParam* pOutput); } SScalarFunctionInfo; typedef struct SMultiFunctionsDesc { @@ -241,7 +253,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI * @param len * @return */ -int32_t qIsBuiltinFunction(const char* name, int32_t len); +int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction); bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index ff28a03260..f79ad35862 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -26,29 +26,6 @@ extern "C" { #include "tvariant.h" #include "function.h" -typedef struct SColumn { - uint64_t tableUid; - int32_t columnIndex; - SColumnInfo info; -} SColumn; - - - -//typedef struct SInterval { -// int32_t tz; // query client timezone -// char intervalUnit; -// char slidingUnit; -// char offsetUnit; -// int64_t interval; -// int64_t sliding; -// int64_t offset; -//} SInterval; -// -//typedef struct SSessionWindow { -// int64_t gap; // gap between two session window(in microseconds) -// int32_t primaryColId; // primary timestamp column -//} SSessionWindow; - typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; @@ -89,12 +66,6 @@ typedef struct STagCond { typedef struct STableMetaInfo { STableMeta *pTableMeta; // table meta, cached in client side and acquired by name SVgroupsInfo *vgroupList; - - /* - * 1. keep the vgroup index during the multi-vnode super table projection query - * 2. keep the vgroup index for multi-vnode insertion - */ - int32_t vgroupIndex; SName name; char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql SArray *tagColList; // SArray, involved tag columns diff --git a/include/util/tdef.h b/include/util/tdef.h index c7a421a55f..652d329cd3 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -134,14 +134,14 @@ do { \ #define TSDB_BINARY_OP_REMAINDER 4004 #define TSDB_BINARY_OP_CONCAT 4005 -#define TSDB_UNARY_OP_CEIL 4500 -#define TSDB_UNARY_OP_FLOOR 4501 -#define TSDB_UNARY_OP_ABS 4502 -#define TSDB_UNARY_OP_ROUND 4503 +#define FUNCTION_CEIL 4500 +#define FUNCTION_FLOOR 4501 +#define FUNCTION_ABS 4502 +#define FUNCTION_ROUND 4503 -#define TSDB_UNARY_OP_LEN 4600 -#define TSDB_UNARY_OP_LTRIM 4601 -#define TSDB_UNARY_OP_RTRIM 4601 +#define FUNCTION_LENGTH 4800 +#define FUNCTION_LTRIM 4801 +#define FUNCTION_RTRIM 4802 #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9b3cbcf9cc..aa7ea48432 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1040,13 +1040,14 @@ static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY } static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); + static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); + setBlockStatisInfo(&pCtx[i], pBlock, NULL/*&pOperator->pExpr[i].base.colInfo*/); } } @@ -2358,8 +2359,8 @@ bool onlyQueryTags(SQueryAttr* pQueryAttr) { if (functionId != FUNCTION_TAGPRJ && functionId != FUNCTION_TID_TAG && - (!(functionId == FUNCTION_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.colInfo.flag)))) { + (!(functionId == FUNCTION_COUNT && pExprInfo->base.pColumns->colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.pColumns->flag)))) { return false; } } @@ -2878,7 +2879,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData int32_t numOfOutput = pTableScanInfo->numOfOutput; for (int32_t i = 0; i < numOfOutput; ++i) { int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; + int32_t colId = pTableScanInfo->pExpr[i].base.pColumns->colId; // group by + first/last should not apply the first/last block filter if (functionId < 0) { @@ -3205,12 +3206,12 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt SExprInfo* pLocalExprInfo = &pExpr[idx]; // ts_comp column required the tag value for join filter - if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { + if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.pColumns->flag)) { continue; } // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type, + doSetTagValueInParam(pTable, pLocalExprInfo->base.pColumns->colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type, pLocalExprInfo->base.resSchema.bytes); if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) @@ -5071,7 +5072,7 @@ SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); // FUNCTION_TAG_DUMMY function needs to be ignored - if (index->colId == pExpr->colInfo.colId && + if (index->colId == pExpr->pColumns->info.colId && ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && functionId == FUNCTION_TAG) || (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && functionId == FUNCTION_PRJ))) { index->colIndex = j; @@ -5290,7 +5291,7 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for(int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData col = {{0}}; - col.info.colId = pExpr[i].base.colInfo.colId; + col.info.colId = pExpr[i].base.pColumns->colId; // col.info.bytes = pExpr[i].base.colBytes; // col.info.type = pExpr[i].base.colType; taosArrayPush(pDataBlock->pDataBlock, &col); @@ -6776,7 +6777,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { int16_t type = pExprInfo->base.resSchema.type; for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.colInfo.colId) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->colId) { bytes = pQueryAttr->tagColList[i].bytes; type = pQueryAttr->tagColList[i].type; break; @@ -6808,10 +6809,10 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { output += sizeof(pQueryAttr->vgId); char* data = NULL; - if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pExprInfo->base.pColumns->colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->colId, type, bytes); } doSetTagValueToResultBuf(output, data, type, bytes); @@ -6839,7 +6840,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { continue; } @@ -6847,10 +6848,10 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { type = pExprInfo[j].base.resSchema.type; bytes = pExprInfo[j].base.resSchema.bytes; - if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pExprInfo[j].base.pColumns->colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->colId, type, bytes); } dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; @@ -7310,9 +7311,9 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { param->pExpr[i] = pExprMsg; - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); +// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); +// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); // pExprMsg->colBytes = htons(pExprMsg->colBytes); // pExprMsg->colType = htons(pExprMsg->colType); @@ -7361,9 +7362,9 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { param->pSecExpr[i] = pExprMsg; - pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); - pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); +// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); +// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); +// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); // pExprMsg->resType = htons(pExprMsg->resType); // pExprMsg->resBytes = htons(pExprMsg->resBytes); // pExprMsg->colBytes = htons(pExprMsg->colBytes); @@ -7677,11 +7678,11 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp SSchema s = {.type=TSDB_DATA_TYPE_BINARY, .bytes=TSDB_MAX_BINARY_LEN}; type = s.type; bytes = s.bytes; - } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && functionId == FUNCTION_TAGPRJ) { // parse the normal column + } else if (pExprs[i].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX && functionId == FUNCTION_TAGPRJ) { // parse the normal column SSchema* s = tGetTbnameColumnSchema(); type = s->type; bytes = s->bytes; - } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.colInfo.colId > TSDB_RES_COL_ID) { + } else if (pExprs[i].base.pColumns->info.colId <= TSDB_UD_COLUMN_INDEX && pExprs[i].base.pColumns->info.colId > TSDB_RES_COL_ID) { // it is a user-defined constant value column assert(functionId == FUNCTION_PRJ); @@ -7692,7 +7693,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp } } else { int32_t j = getColumnIndexInSource(pTableInfo, &pExprs[i].base, pTagCols); - if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { + if (TSDB_COL_IS_TAG(pExprs[i].base.pColumns->flag)) { if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pTableInfo->numOfTags) { tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; @@ -7704,8 +7705,8 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp } } - if (pExprs[i].base.colInfo.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { - SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag))? &pTagCols[j]:&pTableInfo->colList[j]; + if (pExprs[i].base.pColumns->info.colId != TSDB_TBNAME_COLUMN_INDEX && j >= 0) { + SColumnInfo* pCol = (TSDB_COL_IS_TAG(pExprs[i].base.pColumns->flag))? &pTagCols[j]:&pTableInfo->colList[j]; type = pCol->type; bytes = pCol->bytes; } else { @@ -7814,7 +7815,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu // bytes = tDataTypes[type].bytes; // } else { // int32_t index = pExprs[i].base.colInfo.colIndex; -// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.colInfo.colId); +// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.pColumns->colId); // // type = prevExpr[index].base.resSchema.type; // bytes = prevExpr[index].base.resSchema.bytes; @@ -8083,7 +8084,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S pQueryAttr->resultRowSize += pExprs[col].base.resSchema.bytes; // keep the tag length - if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { + if (TSDB_COL_IS_TAG(pExprs[col].base.pColumns->flag)) { pQueryAttr->tagLen += pExprs[col].base.resSchema.bytes; } diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/function/inc/tscalarfunction.h index d8e4c1eeaf..ddb56518e0 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/function/inc/tscalarfunction.h @@ -37,7 +37,7 @@ typedef struct SScalarFunctionSupport { char** data; } SScalarFunctionSupport; -extern struct SScalarFunctionInfo scalarFunc[1]; +extern struct SScalarFunctionInfo scalarFunc[5]; int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index aac4eb1f6d..aa10523dea 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -543,8 +543,8 @@ struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, co pFillCol[i].col.offset = offset; pFillCol[i].col.colId = pExprInfo->base.resSchema.colId; pFillCol[i].tagIndex = -2; - pFillCol[i].flag = pExprInfo->base.colInfo.flag; // always be the normal column for table query - pFillCol[i].functionId = pExprInfo->pExpr->_node.functionId; + pFillCol[i].flag = pExprInfo->base.pColumns->flag; // always be the normal column for table query + pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; pFillCol[i].fillVal.i = fillVal[i]; offset += pExprInfo->base.resSchema.bytes; diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index e9a59f7d69..c47f3a249a 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -30,11 +30,12 @@ static void doInitFunctionHashTable() { static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT; -int32_t qIsBuiltinFunction(const char* name, int32_t len) { +int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction) { pthread_once(&functionHashTableInit, doInitFunctionHashTable); SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len); if (pInfo != NULL) { + *scalarFunction = ((*pInfo)->type == FUNCTION_TYPE_SCALAR); return (*pInfo)->functionId; } else { return -1; diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 8c9d670e87..80d426f101 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -2,6 +2,156 @@ #include "tbinoperator.h" #include "tunaryoperator.h" + +static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { + dst->type = src->type; + dst->bytes = src->bytes; + dst->num = src->num; +} + +static void tceil(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = ceilf(p[i]); + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*)pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = ceil(p[i]); + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void tfloor(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = floorf(p[i]); + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*) pOutput->data; + + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = floor(p[i]); + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void _tabs(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_TINYINT: { + int8_t* p = (int8_t*) pLeft->data; + int8_t* out = (int8_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* p = (int16_t*) pLeft->data; + int16_t* out = (int16_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_INT: { + int32_t* p = (int32_t*) pLeft->data; + int32_t* out = (int32_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* p = (int64_t*) pLeft->data; + int64_t* out = (int64_t*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = (p[i] > 0)? p[i]:-p[i]; + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void tround(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + assignBasicParaInfo(pOutput, pLeft); + + switch (pLeft->bytes) { + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) pLeft->data; + float* out = (float*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = roundf(p[i]); + } + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) pLeft->data; + double* out = (double*) pOutput->data; + for (int32_t i = 0; i < pLeft->num; ++i) { + out[i] = round(p[i]); + } + } + + default: + memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); + } +} + +static void tlength(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { + int64_t* out = (int64_t*) pOutput->data; + char* s = pLeft->data; + + for(int32_t i = 0; i < pLeft->num; ++i) { + out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); + } +} + static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: @@ -168,11 +318,12 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa return 0; } -SScalarFunctionInfo scalarFunc[1] = { - { - - }, - +SScalarFunctionInfo scalarFunc[5] = { + {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, + {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor}, + {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs}, + {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround}, + {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength}, }; void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { diff --git a/source/libs/function/src/tunaryoperator.c b/source/libs/function/src/tunaryoperator.c index 9105942d26..9651f98d08 100644 --- a/source/libs/function/src/tunaryoperator.c +++ b/source/libs/function/src/tunaryoperator.c @@ -1,171 +1,13 @@ #include "tunaryoperator.h" -static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScalarFuncParam* src) { - dst->type = src->type; - dst->bytes = src->bytes; - dst->num = src->num; -} -static void tceil(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { - assignBasicParaInfo(pOutput, pLeft); - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = ceilf(p[i]); - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*)pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = ceil(p[i]); - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); - } -} - -static void tfloor(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { - assignBasicParaInfo(pOutput, pLeft); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = floorf(p[i]); - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = floor(p[i]); - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); - } -} - -static void _tabs(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { - assignBasicParaInfo(pOutput, pLeft); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t* p = (int8_t*) pLeft->data; - int8_t* out = (int8_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t* p = (int16_t*) pLeft->data; - int16_t* out = (int16_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_INT: { - int32_t* p = (int32_t*) pLeft->data; - int32_t* out = (int32_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - case TSDB_DATA_TYPE_BIGINT: { - int64_t* p = (int64_t*) pLeft->data; - int64_t* out = (int64_t*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = (p[i] > 0)? p[i]:-p[i]; - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); - } -} - -static void tround(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { - assignBasicParaInfo(pOutput, pLeft); - - switch (pLeft->bytes) { - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) pLeft->data; - float* out = (float*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = roundf(p[i]); - } - } - - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) pLeft->data; - double* out = (double*) pOutput->data; - for (int32_t i = 0; i < pLeft->num; ++i) { - out[i] = round(p[i]); - } - } - - default: - memcpy(pOutput->data, pLeft->data, pLeft->num* pLeft->bytes); - } -} - -static void tlen(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { - int64_t* out = (int64_t*) pOutput->data; - char* s = pLeft->data; - - for(int32_t i = 0; i < pLeft->num; ++i) { - out[i] = varDataLen(POINTER_SHIFT(s, i * pLeft->bytes)); - } -} +// TODO dynamic define these functions _unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) { - switch(operator) { - case TSDB_UNARY_OP_CEIL: - return tceil; - case TSDB_UNARY_OP_FLOOR: - return tfloor; - case TSDB_UNARY_OP_ROUND: - return tround; - case TSDB_UNARY_OP_ABS: - return _tabs; - case TSDB_UNARY_OP_LEN: - return tlen; - default: - assert(0); - } + assert(0); } bool isStringOperatorFn(int32_t op) { - return op == TSDB_UNARY_OP_LEN; + return op == FUNCTION_LENGTH; } diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 922b9c2d44..0f5acf5bcc 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -38,6 +38,7 @@ extern "C" { TAOS_FIELD createField(const SSchema* pSchema); SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); +void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); SInternalField* insertFieldInfo(SFieldInfo* pFieldInfo, int32_t index, SSchema* field); int32_t getNumOfFields(SFieldInfo* pFieldInfo); @@ -51,7 +52,7 @@ STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo); void columnListCopyAll(SArray* dst, const SArray* src); -SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema); +SColumn* columnListInsert(SArray* pColumnList, uint64_t uid, SSchema* pSchema, int32_t flag); SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid); void cleanupTagCond(STagCond* pTagCond); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 26ab04cb85..c25e647b48 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -327,7 +327,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SQueryStmtI int32_t startOffset = (int32_t) taosArrayGetSize(pQueryInfo->colList); for(int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { - columnListInsert(pQueryInfo->colList, i + startOffset, pMeta->uid, &pMeta->schema[i]); + columnListInsert(pQueryInfo->colList, pMeta->uid, &pMeta->schema[i], TSDB_COL_NORMAL); } return TSDB_CODE_SUCCESS; @@ -556,14 +556,14 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, SMsgBuf* taosArrayPush(pGroupExpr->columnInfo, &colIndex); index.columnIndex = relIndex; - columnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->uid, pSchema); + columnListInsert(pTableMetaInfo->tagColList, pTableMeta->uid, pSchema, colIndex.flag); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { return buildInvalidOperationMsg(pMsgBuf, msg5); } - columnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->uid, pSchema); + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); @@ -854,7 +854,7 @@ int32_t validateStateWindowNode(SQueryStmtInfo *pQueryInfo, SWindowStateVal* pWi return buildInvalidOperationMsg(pMsgBuf, msg2); } - columnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->uid, pSchema); + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; //TODO use group by routine? state window query not support stable query. @@ -1114,7 +1114,7 @@ int32_t checkForInvalidOrderby(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, S pExpr = getExprInfo(pQueryInfo, pos); // other tag are not allowed - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pExpr->base.pColumns->colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_ID) { return buildInvalidOperationMsg(pMsgBuf, msg5); } @@ -1133,7 +1133,7 @@ int32_t checkForInvalidOrderby(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, S bool found = false; for (int32_t i = 0; i < getNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = getExprInfo(pQueryInfo, i); - if (getExprFunctionId(pExpr) == FUNCTION_PRJ && pExpr->base.colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (getExprFunctionId(pExpr) == FUNCTION_PRJ && pExpr->base.pColumns->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { found = true; break; } @@ -1214,7 +1214,7 @@ int32_t checkForInvalidOrderby(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, S pExpr = getExprInfo(pQueryInfo, pos); - if (pExpr->base.colInfo.colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pExpr->base.pColumns->colIndex != index.columnIndex && index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_ID) { return buildInvalidOperationMsg(pMsgBuf, msg5); } } @@ -1614,7 +1614,7 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { for (int32_t i = 0; i < size; ++i) { SExprInfo* pExpr = getExprInfo(pQueryInfo, i); int32_t functionId = getExprFunctionId(pExpr); - if (functionId == FUNCTION_COUNT && TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) { + if (functionId == FUNCTION_COUNT && TSDB_COL_IS_TAG(pExpr->base.pColumns->flag)) { return buildInvalidOperationMsg(pMsgBuf, msg1); } } @@ -1718,15 +1718,15 @@ SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, uint64_t uid = pTableMetaInfo->pTableMeta->uid; SArray* p = TSDB_COL_IS_TAG(pIndex->type)?pTableMetaInfo->tagColList:pQueryInfo->colList; - columnListInsert(p, pIndex->columnIndex, uid, pColSchema); + columnListInsert(p, uid, pColSchema, pIndex->type); - pExpr->base.colInfo.flag = pIndex->type; + pExpr->base.pColumns->flag = pIndex->type; if (TSDB_COL_IS_NORMAL_COL(pIndex->type)) { insertPrimaryTsColumn(pQueryInfo->colList, uid); } if (finalResult) { - addResColumnInfo(pQueryInfo, outputColIndex, pColSchema, pExpr); + addResColumnInfo(pQueryInfo, outputColIndex, pResultSchema, pExpr); } return pExpr; @@ -1762,7 +1762,7 @@ static int32_t checkForAliasName(SMsgBuf* pMsgBuf, char* aliasName) { } static int32_t validateComplexExpr(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, SArray* pColList, int32_t* type, SMsgBuf* pMsgBuf); -static int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, uint64_t *uid, SMsgBuf* pMsgBuf); +static int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, SMsgBuf* pMsgBuf); static int64_t getTickPerSecond(SVariant* pVariant, int32_t precision, int64_t* tickPerSec, SMsgBuf *pMsgBuf) { const char* msg10 = "derivative duration should be greater than 1 Second"; @@ -1795,7 +1795,7 @@ static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTab addExprInfo(pQueryInfo, outputIndex, pExpr); SSchema* pSourceSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, indexTS.columnIndex); - columnListInsert(pQueryInfo->colList, indexTS.columnIndex, pTableMetaInfo->pTableMeta->uid, pSourceSchema); + columnListInsert(pQueryInfo->colList, pTableMetaInfo->pTableMeta->uid, pSourceSchema, TSDB_COL_NORMAL); addResColumnInfo(pQueryInfo, outputIndex, &pExpr->base.resSchema, pExpr); } @@ -1898,6 +1898,7 @@ static int32_t doHandleOneParam(SQueryStmtInfo *pQueryInfo, tSqlExprItem* pItem, } static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumnList, SMsgBuf* pMsgBuf); +static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf); int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId, STableMetaInfo** pTableMetaInfo, SSchema* columnSchema, tExprNode** pNode, SColumnIndex* pIndex, tSqlExprItem* pParamElem, SMsgBuf* pMsgBuf) { @@ -1906,24 +1907,47 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId const char* msg3 = "illegal column name"; const char* msg4 = "nested function is not supported"; const char* msg5 = "functions applied to tags are not allowed"; + const char* msg6 = "aggregate function can not be nested in aggregate function"; + const char* msg7 = "invalid function name"; if (tokenId == TK_ALL || tokenId == TK_ID) { // simple parameter - if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, pIndex, pMsgBuf) != TSDB_CODE_SUCCESS)) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + // simple parameter or nested function + // It is a parameter of a aggregate function, so it can not be still a aggregate function. + // E.g., the sql statement of "select count(count(*)) from table_name" is invalid. + tSqlExpr* pSqlExpr = pParamElem->pNode; + if (pParamElem->pNode->type == SQL_NODE_SQLFUNCTION) { + bool scalarFunc = false; + pParamElem->functionId = qIsBuiltinFunction(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n, &scalarFunc); + if (pParamElem->functionId == FUNCTION_INVALID_ID) { + return buildInvalidOperationMsg(pMsgBuf, msg7); + } + + if (!scalarFunc) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + int32_t code = createComplexExpr(pQueryInfo, i, pParamElem, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + } else { + if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, pIndex, pMsgBuf) != TSDB_CODE_SUCCESS)) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + // functions can not be applied to tags + if (TSDB_COL_IS_TAG(pIndex->type)) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + // 2. check if sql function can be applied on this column data type + *pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); + *columnSchema = *(SSchema*)getOneColumnSchema((*pTableMetaInfo)->pTableMeta, pIndex->columnIndex); } - - // functions can not be applied to tags - if (TSDB_COL_IS_TAG(pIndex->type)) { - return buildInvalidOperationMsg(pMsgBuf, msg5); - } - - *pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); - - // 2. check if sql function can be applied on this column data type - *columnSchema = *(SSchema*) getOneColumnSchema((*pTableMetaInfo)->pTableMeta, pIndex->columnIndex); - } else if (tokenId == TK_PLUS || tokenId == TK_MINUS || tokenId == TK_STAR || tokenId == TK_REM || tokenId == TK_DIVIDE || tokenId == TK_CONCAT) { + }else if (tokenId == TK_PLUS || tokenId == TK_MINUS || tokenId == TK_STAR || tokenId == TK_REM || tokenId == TK_DIVIDE || tokenId == TK_CONCAT) { int32_t arithmeticType = NON_ARITHMEIC_EXPR; - SArray* pColumnList = taosArrayInit(4, sizeof(SColumnIndex)); + SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); if (validateComplexExpr(pParamElem->pNode, pQueryInfo, pColumnList, &arithmeticType, pMsgBuf) != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } @@ -1940,7 +1964,7 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId tstrncpy(columnSchema->name, pExprToken->z, len); SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = sqlExprToExprNode(pNode, pParamElem->pNode, pQueryInfo, colList, NULL, pMsgBuf); + int32_t ret = sqlExprToExprNode(pNode, pParamElem->pNode, pQueryInfo, colList, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); tExprTreeDestroy(*pNode, NULL); @@ -2290,7 +2314,7 @@ int32_t addExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSqlEx return buildInvalidOperationMsg(pMsgBuf, msg1); } - columnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMetaInfo->pTableMeta->uid, &pSchema[index.columnIndex]); + columnListInsert(pTableMetaInfo->tagColList, pTableMetaInfo->pTableMeta->uid, &pSchema[index.columnIndex], TSDB_COL_TAG); SSchema* pTagSchema = getTableTagSchema(pTableMetaInfo->pTableMeta); SSchema s = {0}; @@ -2337,6 +2361,8 @@ int32_t addExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSqlEx return TSDB_CODE_SUCCESS; } + + default: { // pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); // if (pUdfInfo == NULL) { @@ -2380,6 +2406,110 @@ int32_t addExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSqlEx return TSDB_CODE_TSC_INVALID_OPERATION; } +static int32_t validateExprLeafColumnNode(SQueryStmtInfo *pQueryInfo, SToken* pColumnName, SArray* pList, SMsgBuf* pMsgBuf) { + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(pColumnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // if column is timestamp not support arithmetic, so return invalid sql + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); + if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + SColumn c = {0}; + setColumn(&c, pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); + taosArrayPush(pList, &c); + + return TSDB_CODE_SUCCESS; +} + +static int32_t validateExprLeafFunctionNode(SQueryStmtInfo* pQueryInfo, tSqlExpr* pExpr, SMsgBuf* pMsgBuf) { + tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; + + // sql function list in selection clause. + // Append the sqlExpr into exprList of pQueryInfo structure sequentially + bool scalar = false; + item.functionId = qIsBuiltinFunction(pExpr->Expr.operand.z, pExpr->Expr.operand.n, &scalar); + if (item.functionId < 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); + if (addExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) + int32_t inc = (int32_t) getNumOfExprs(pQueryInfo) - outputIndex; + if (inc > 1) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // Not supported data type in expression + for(int32_t i = 0; i < inc; ++i) { + SExprInfo* p1 = getExprInfo(pQueryInfo, i + outputIndex); + int16_t t = p1->base.resSchema.type; + if (t == TSDB_DATA_TYPE_TIMESTAMP) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t validateScalarFunctionParamNum(tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { + int32_t code = TSDB_CODE_SUCCESS; + switch (pItem->functionId) { + case FUNCTION_CEIL: { + code = checkForkParam(pItem->pNode, 1, pMsgBuf); + break; + } + case FUNCTION_LENGTH: { + code = checkForkParam(pItem->pNode, 1, pMsgBuf); + break; + } + } + + return code; +} + +int32_t validateScalarFunctionParam(SQueryStmtInfo* pQueryInfo, tSqlExpr* pExpr, SArray* pList, int32_t* exprType, SMsgBuf* pMsgBuf) { + int32_t code = TSDB_CODE_SUCCESS; + // more than one parameter for count() function + + SArray* pParamList = pExpr->Expr.paramList; + *exprType = NORMAL_ARITHMETIC; + + for (int32_t i = 0; i < 1; ++i) { + tSqlExprItem* pParamElem = taosArrayGet(pParamList, i); + tSqlExpr* pSqlExpr = pParamElem->pNode; + + int32_t type = pSqlExpr->type; + if (type == SQL_NODE_VALUE) { + } else if (type == SQL_NODE_SQLFUNCTION) { + code = validateExprLeafFunctionNode(pQueryInfo, pSqlExpr, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else if (type == SQL_NODE_EXPR) { + code = validateComplexExpr(pSqlExpr, pQueryInfo, pList, exprType, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else if (type == SQL_NODE_TABLE_COLUMN) { + code = validateExprLeafColumnNode(pQueryInfo, &pSqlExpr->columnName, pList, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } +} + SExprInfo* doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, SColumnIndex* pColIndex, const char* aliasName, int32_t colId) { STableMeta* pTableMeta = getMetaInfo(pQueryInfo, pColIndex->tableIndex)->pTableMeta; @@ -2452,6 +2582,48 @@ static SSchema createConstantColumnSchema(SVariant* pVal, const SToken* exprStr, return s; } +static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, SColumnIndex* pIndex, int32_t startPos, bool outerQuery, SMsgBuf* pMsgBuf) { + const char* msg3 = "tbname not allowed in outer query"; + + SSchema colSchema = {0}; + int32_t functionId = 0; + + if (outerQuery) { // todo?? + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); + + bool existed = false; + SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; + + int32_t numOfCols = getNumOfColumns(pTableMetaInfo->pTableMeta); + for (int32_t i = 0; i < numOfCols; ++i) { + if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { + existed = true; + pIndex->columnIndex = i; + break; + } + } + + if (!existed) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + colSchema = pSchema[pIndex->columnIndex]; + functionId = FUNCTION_PRJ; + } else { + colSchema = *getTbnameColumnSchema(); + functionId = FUNCTION_TAGPRJ; + } + + SSchema resultSchema = colSchema; + resultSchema.colId = getNewResColId(); + + char rawName[TSDB_COL_NAME_LEN] = {0}; + setTokenAndResColumnName(pItem, resultSchema.name, rawName, sizeof(colSchema.name) - 1); + + doAddOneExprInfo(pQueryInfo, startPos, functionId, pIndex, &colSchema, &resultSchema, NULL, 0, rawName, true); + return TSDB_CODE_SUCCESS; +} + int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery, SMsgBuf* pMsgBuf) { const char* msg1 = "tag for normal table query is not allowed"; const char* msg2 = "invalid column name"; @@ -2487,7 +2659,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* if (pTableMeta->tableType != TSDB_TEMP_TABLE) { insertPrimaryTsColumn(pQueryInfo->colList, pTableMeta->uid); } - } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { // simple column projection query + } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { //constant value column SColumnIndex index = createConstantColumnIndex(&pQueryInfo->udColumnId); SSchema colSchema = createConstantColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); @@ -2498,49 +2670,14 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; taosVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); - } else if (tokenId == TK_ID) { + } else if (tokenId == TK_ID) { // column name SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg2); } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = {0}; - int32_t functionId = 0; - - if (outerQuery) { // todo?? - STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); - - bool existed = false; - SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; - - int32_t numOfCols = getNumOfColumns(pTableMetaInfo->pTableMeta); - for (int32_t i = 0; i < numOfCols; ++i) { - if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { - existed = true; - index.columnIndex = i; - break; - } - } - - if (!existed) { - return buildInvalidOperationMsg(pMsgBuf, msg3); - } - - colSchema = pSchema[index.columnIndex]; - functionId = FUNCTION_PRJ; - } else { - colSchema = *getTbnameColumnSchema(); - functionId = FUNCTION_TAGPRJ; - } - - SSchema resultSchema = colSchema; - resultSchema.colId = getNewResColId(); - - char rawName[TSDB_COL_NAME_LEN] = {0}; - setTokenAndResColumnName(pItem, resultSchema.name, rawName, sizeof(colSchema.name) - 1); - - doAddOneExprInfo(pQueryInfo, startPos, functionId, &index, &colSchema, &resultSchema, NULL, 0, rawName, true); + handleTbnameProjection(pQueryInfo, pItem, &index, startPos, outerQuery, pMsgBuf); } else { STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); if (TSDB_COL_IS_TAG(index.type) && UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { @@ -2562,8 +2699,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* return TSDB_CODE_SUCCESS; } -static int32_t validateExprLeafNode(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, SArray* pList, int32_t* type, uint64_t* uid, - SMsgBuf* pMsgBuf) { +static int32_t validateExprLeafNode(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, SArray* pList, int32_t* type, SMsgBuf* pMsgBuf) { if (pExpr->type == SQL_NODE_TABLE_COLUMN) { if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; @@ -2571,21 +2707,10 @@ static int32_t validateExprLeafNode(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, return TSDB_CODE_TSC_INVALID_OPERATION; } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (getColumnIndexByName(&pExpr->columnName, pQueryInfo, &index, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t code = validateExprLeafColumnNode(pQueryInfo, &pExpr->columnName, pList, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; } - - // if column is timestamp, bool, binary, nchar, not support arithmetic, so return invalid sql - STableMeta* pTableMeta = getMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; - - SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); - if ((pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) || (pSchema->type == TSDB_DATA_TYPE_BOOL) || - (pSchema->type == TSDB_DATA_TYPE_BINARY) || (pSchema->type == TSDB_DATA_TYPE_NCHAR)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - taosArrayPush(pList, &index); } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.d) || isinf(pExpr->value.d))) || pExpr->tokenId == TK_NULL) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -2596,103 +2721,99 @@ static int32_t validateExprLeafNode(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, return TSDB_CODE_TSC_INVALID_OPERATION; } - tSqlExprItem item = {.pNode = pExpr, .aliasName = NULL}; - - // sql function list in selection clause. - // Append the sqlExpr into exprList of pQueryInfo structure sequentially - item.functionId = qIsBuiltinFunction(pExpr->Expr.operand.z, pExpr->Expr.operand.n); - if (item.functionId < 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t code = validateExprLeafFunctionNode(pQueryInfo, pExpr, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; } - - int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); - if (addExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) - int32_t inc = (int32_t) getNumOfExprs(pQueryInfo) - outputIndex; - if (inc > 1) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // Not supported data type in arithmetic expression - uint64_t id = -1; - for(int32_t i = 0; i < inc; ++i) { - SExprInfo* p1 = getExprInfo(pQueryInfo, i + outputIndex); - - int16_t t = p1->base.resSchema.type; - if (IS_VAR_DATA_TYPE(t) || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - if (i == 0) { - id = p1->base.uid; - continue; - } - - if (id != p1->base.uid) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - *uid = id; } return TSDB_CODE_SUCCESS; } -int32_t validateComplexExpr(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, SArray* pColList, int32_t* type, SMsgBuf* pMsgBuf) { +int32_t validateComplexExpr(tSqlExpr * pExpr, SQueryStmtInfo* pQueryInfo, SArray* pColList, int32_t* type, SMsgBuf* pMsgBuf) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - tSqlExpr* pLeft = pExpr->pLeft; - uint64_t uidLeft = 0; - uint64_t uidRight = 0; + int32_t code = TSDB_CODE_SUCCESS; + if (pExpr->type == SQL_NODE_SQLFUNCTION) { + return validateScalarFunctionParam(pQueryInfo, pExpr, pColList, type, pMsgBuf); + } + tSqlExpr* pLeft = pExpr->pLeft; if (pLeft->type == SQL_NODE_EXPR) { - int32_t ret = validateComplexExpr(pLeft, pQueryInfo, pColList, type, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + code = validateComplexExpr(pLeft, pQueryInfo, pColList, type, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; } } else { - int32_t ret = validateExprLeafNode(pLeft, pQueryInfo, pColList, type, &uidLeft, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + code = validateExprLeafNode(pLeft, pQueryInfo, pColList, type, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; } } tSqlExpr* pRight = pExpr->pRight; if (pRight->type == SQL_NODE_EXPR) { - int32_t ret = validateComplexExpr(pRight, pQueryInfo, pColList, type, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + code = validateComplexExpr(pRight, pQueryInfo, pColList, type, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; } } else { - int32_t ret = validateExprLeafNode(pRight, pQueryInfo, pColList, type, &uidRight, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + code = validateExprLeafNode(pRight, pQueryInfo, pColList, type, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + // check divide by 0 + if (pExpr->tokenId == TK_DIVIDE && pRight->type == SQL_NODE_VALUE) { + int32_t type1 = pRight->value.nType; + const char* msg1 = "invalid expr (divide by 0)"; + + if (type1 == TSDB_DATA_TYPE_DOUBLE && pRight->value.d < DBL_EPSILON) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } else if (type1 == TSDB_DATA_TYPE_INT && pRight->value.i == 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } } return TSDB_CODE_SUCCESS; } -int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, uint64_t *uid, SMsgBuf* pMsgBuf) { +int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, SMsgBuf* pMsgBuf) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; + if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { + // assert it is a scalar function + *pExpr = calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; + (*pExpr)->_function.num = 1; + (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); + + SArray* pParamList = pSqlExpr->Expr.paramList; + size_t num = taosArrayGetSize(pParamList); + (*pExpr)->_function.pChild = calloc(num, POINTER_BYTES); + + for(int32_t i = 0; i < num; ++i) { + tSqlExprItem* pItem = taosArrayGet(pParamList, i); + sqlExprToExprNode(&(((*pExpr)->_function.pChild)[0]), pItem->pNode, pQueryInfo, pCols, pMsgBuf); + } + + return TSDB_CODE_SUCCESS; + } + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pSqlExpr->pLeft != NULL) { - int32_t ret = sqlExprToExprNode(&pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid, pMsgBuf); + int32_t ret = sqlExprToExprNode(&pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } } if (pSqlExpr->pRight != NULL) { - int32_t ret = sqlExprToExprNode(&pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid, pMsgBuf); + int32_t ret = sqlExprToExprNode(&pRight, pSqlExpr->pRight, pQueryInfo, pCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(pLeft, NULL); return ret; @@ -2728,7 +2849,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } return ret; } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { - // arithmetic expression on the results of aggregation functions + // Expression on the results of aggregation functions *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TEXPR_COL_NODE; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); @@ -2737,19 +2858,28 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt // set the input column data byte and type. size_t size = taosArrayGetSize(pQueryInfo->exprList); + bool found = false; + uint64_t uid = 0; for (int32_t i = 0; i < size; ++i) { SExprInfo* p1 = taosArrayGetP(pQueryInfo->exprList, i); if (strcmp((*pExpr)->pSchema->name, p1->base.resSchema.name) == 0) { memcpy((*pExpr)->pSchema, &p1->base.resSchema, sizeof(SSchema)); - if (uid != NULL) { - *uid = p1->base.uid; - } - + found = true; + uid = p1->base.pColumns->uid; break; } } - } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression + + assert(found); + + if (pCols != NULL) { // record the involved columns + SColumn c = {0}; + setColumn(&c, uid, NULL, TSDB_COL_NORMAL, (*pExpr)->pSchema); + taosArrayPush(pCols, &c); + } + + } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column expression int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -2764,17 +2894,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; - - if (pCols != NULL) { // record the involved columns - SColIndex colIndex = {0}; - tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name)); - colIndex.colId = pSchema->colId; - colIndex.colIndex = index.columnIndex; - colIndex.flag = index.type; - - taosArrayPush(pCols, &colIndex); - } - return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { int32_t colType = -1; @@ -2823,17 +2942,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt assert((*pExpr)->_node.optr != 0); - // check for dividing by 0 - if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) { - if (pRight->nodeType == TEXPR_VALUE_NODE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i == 0) { - return buildInvalidOperationMsg(pMsgBuf, "invalid expr (divide by 0)"); - } else if (pRight->pVal->nType == TSDB_DATA_TYPE_FLOAT && pRight->pVal->d == 0) { - return buildInvalidOperationMsg(pMsgBuf, "invalid expr (divide by 0)"); - } - } - } - // NOTE: binary|nchar data allows the >|< type filter if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { if (pRight != NULL && pRight->nodeType == TEXPR_VALUE_NODE) { @@ -2848,33 +2956,32 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumnList, SMsgBuf* pMsgBuf) { - const char* msg3 = "tag columns can not be used in arithmetic expression"; + const char* msg1 = "tag can not be used in expression"; - SColumnIndex* p1 = taosArrayGet(pColumnList, 0); - STableMeta* pTableMeta = getMetaInfo(pQueryInfo, p1->tableIndex)->pTableMeta; + SColumn* p1 = taosArrayGet(pColumnList, 0); size_t numOfNode = taosArrayGetSize(pColumnList); for(int32_t k = 0; k < numOfNode; ++k) { - SColumnIndex* pIndex = taosArrayGet(pColumnList, k); - if (TSDB_COL_IS_TAG(pIndex->type)) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + SColumn* p = taosArrayGet(pColumnList, k); + if (TSDB_COL_IS_TAG(p->flag)) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } - SSchema* ps = getOneColumnSchema(pTableMeta, pIndex->columnIndex); - columnListInsert(pQueryInfo->colList, pIndex->columnIndex, pTableMeta->uid, ps); + SSchema s = createSchema(p->info.type, p->info.bytes, p->info.colId, p->name); + columnListInsert(pQueryInfo->colList, p->uid, &s, p->flag); } - insertPrimaryTsColumn(pQueryInfo->colList, pTableMeta->uid); + insertPrimaryTsColumn(pQueryInfo->colList, p1->uid); return TSDB_CODE_SUCCESS; } static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { - const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; + const char* msg1 = "invalid column name, illegal column type, or columns in expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; - const char* msg3 = "tag columns can not be used in arithmetic expression"; int32_t arithmeticType = NON_ARITHMEIC_EXPR; - SArray* pColumnList = taosArrayInit(4, sizeof(SColumnIndex)); + + SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); if (validateComplexExpr(pItem->pNode, pQueryInfo, pColumnList, &arithmeticType, pMsgBuf) != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } @@ -2884,8 +2991,8 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); tExprNode* pNode = NULL; - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, colList, NULL, pMsgBuf); + SArray* colList = taosArrayInit(10, sizeof(SColumn)); + int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, colList, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); tExprTreeDestroy(pNode, NULL); @@ -2924,13 +3031,13 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tbufCloseWriter(&bw); taosArrayDestroy(colList); } else { - SColumnIndex columnIndex = {0}; - SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); addResColumnInfo(pQueryInfo, exprIndex, &s, NULL); + assert(taosArrayGetSize(pColumnList) == 0); + tExprNode* pNode = NULL; - int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, NULL, NULL, pMsgBuf); + int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(pNode, NULL); return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); @@ -2938,10 +3045,19 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); addExprInfo(pQueryInfo, exprIndex, pExpr); + setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); - pExpr->base.numOfParams = 1; + // extract columns according to the tExprNode tree + size_t num = taosArrayGetSize(pColumnList); + pExpr->base.pColumns = calloc(num, sizeof(SColumn)); + for(int32_t i = 0; i < num; ++i) { + pExpr->base.pColumns[i] = *(SColumn*) taosArrayGet(pColumnList, i); + } + pExpr->base.numOfCols = num; + + pExpr->base.numOfParams = 1; SBufferWriter bw = tbufInitWriter(NULL, false); // TRY(0) { exprTreeToBinary(&bw, pExpr->pExpr); @@ -2979,7 +3095,8 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, return buildInvalidOperationMsg(pMsgBuf, msg1); } - size_t numOfExpr = taosArrayGetSize(pSelNodeList); + int32_t code = TSDB_CODE_SUCCESS; + size_t numOfExpr = taosArrayGetSize(pSelNodeList); for (int32_t i = 0; i < numOfExpr; ++i) { int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); @@ -2987,15 +3104,17 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, int32_t type = pItem->pNode->type; if (pItem->distinct) { - if (i != 0/* || type == SQL_NODE_SQLFUNCTION || type == SQL_NODE_EXPR*/) { + if (i != 0 || type == SQL_NODE_SQLFUNCTION || type == SQL_NODE_EXPR) { return buildInvalidOperationMsg(pMsgBuf, msg4); } pQueryInfo->info.distinct = true; } + bool scalarFunc = false; + if (type == SQL_NODE_SQLFUNCTION) { - pItem->functionId = qIsBuiltinFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); + pItem->functionId = qIsBuiltinFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n, &scalarFunc); if (pItem->functionId == FUNCTION_INVALID_ID) { int32_t functionId = FUNCTION_INVALID_ID; bool valid = qIsValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n, &functionId); @@ -3004,21 +3123,24 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, } pItem->functionId = functionId; + } else if (scalarFunc) { + if ((code = createComplexExpr(pQueryInfo, i, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } } // sql function in selection clause, append sql function info in pSqlCmd structure sequentially - if (addExprAndResColumn(pQueryInfo, outputIndex, pItem, true, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + if ((code = addExprAndResColumn(pQueryInfo, outputIndex, pItem, true, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; } } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResColumn(pQueryInfo, pItem, outerQuery, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; + if ((code = addProjectionExprAndResColumn(pQueryInfo, pItem, outerQuery, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; } } else if (type == SQL_NODE_EXPR) { - int32_t code = createComplexExpr(pQueryInfo, i, pItem, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { + if ((code = createComplexExpr(pQueryInfo, i, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { return code; } } else { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 504971cc10..0585dc8905 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -177,7 +177,8 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet } // Let's assume that it is an UDF/UDAF, if it is not a built-in function. - if (qIsBuiltinFunction(t->z, t->n) < 0) { + bool scalarFunc = false; + if (qIsBuiltinFunction(t->z, t->n, &scalarFunc) < 0) { char* fname = strndup(t->z, t->n); taosArrayPush(pMetaInfo->pUdf, &fname); } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7dc5fd00b2..13148da66b 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -513,6 +513,20 @@ SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* nam return s; } +void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema) { + pColumn->uid = uid; + pColumn->flag = flag; + pColumn->info.colId = pSchema->colId; + pColumn->info.bytes = pSchema->bytes; + pColumn->info.type = pSchema->type; + + if (tableName != NULL) { + snprintf(pColumn->name, tListLen(pColumn->name), "%s.%s", tableName, pSchema->name); + } else { + tstrncpy(pColumn->name, pSchema->name, tListLen(pColumn->name)); + } +} + int32_t getNumOfFields(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } @@ -701,7 +715,7 @@ int32_t columnExists(SArray* pColumnList, int32_t columnId, uint64_t uid) { int32_t i = 0; while (i < numOfCols) { SColumn* pCol = taosArrayGetP(pColumnList, i); - if ((pCol->info.colId != columnId) || (pCol->tableUid != uid)) { + if ((pCol->info.colId != columnId) || (pCol->uid != uid)) { ++i; continue; } else { @@ -716,64 +730,60 @@ int32_t columnExists(SArray* pColumnList, int32_t columnId, uint64_t uid) { return i; } -SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema) { +static int32_t doFindPosition(const SArray* pColumnList, uint64_t uid, const SSchema* pSchema) { + int32_t i = 0; + + size_t numOfCols = taosArrayGetSize(pColumnList); + while (i < numOfCols) { + SColumn* pCol = taosArrayGetP(pColumnList, i); + if (pCol->uid < uid) { + i++; + continue; + } + + if (pCol->info.colId < pSchema->colId) { + i++; + continue; + } + + break; + } + + return i; +} + +SColumn* columnListInsert(SArray* pColumnList, uint64_t uid, SSchema* pSchema, int32_t flag) { // ignore the tbname columnIndex to be inserted into source list - if (columnIndex < 0) { + assert(pSchema != NULL && pColumnList != NULL); + + int32_t i = doFindPosition(pColumnList, uid, pSchema); + size_t size = taosArrayGetSize(pColumnList); + if (size > 0 && i < size) { + SColumn* pCol = taosArrayGetP(pColumnList, i); + if (pCol->uid == uid && pCol->info.colId == pSchema->colId) { + return pCol; + } + } + + SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { return NULL; } - size_t numOfCols = taosArrayGetSize(pColumnList); + b->uid = uid; + b->flag = flag; + b->info.colId = pSchema->colId; + b->info.bytes = pSchema->bytes; + b->info.type = pSchema->type; + tstrncpy(b->name, pSchema->name, tListLen(b->name)); - int32_t i = 0; - while (i < numOfCols) { - SColumn* pCol = taosArrayGetP(pColumnList, i); - if (pCol->columnIndex < columnIndex) { - i++; - } else if (pCol->tableUid < uid) { - i++; - } else { - break; - } - } - - if (i >= numOfCols || numOfCols == 0) { - SColumn* b = calloc(1, sizeof(SColumn)); - if (b == NULL) { - return NULL; - } - - b->columnIndex = columnIndex; - b->tableUid = uid; - b->info.colId = pSchema->colId; - b->info.bytes = pSchema->bytes; - b->info.type = pSchema->type; - - taosArrayInsert(pColumnList, i, &b); - } else { - SColumn* pCol = taosArrayGetP(pColumnList, i); - - if (i < numOfCols && (pCol->columnIndex > columnIndex || pCol->tableUid != uid)) { - SColumn* b = calloc(1, sizeof(SColumn)); - if (b == NULL) { - return NULL; - } - - b->columnIndex = columnIndex; - b->tableUid = uid; - b->info.colId = pSchema->colId; - b->info.bytes = pSchema->bytes; - b->info.type = pSchema->type; - - taosArrayInsert(pColumnList, i, &b); - } - } - - return taosArrayGetP(pColumnList, i); + taosArrayInsert(pColumnList, i, &b); + return b; } SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid) { SSchema s = {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE, .colId = PRIMARYKEY_TIMESTAMP_COL_ID}; - return columnListInsert(pColumnList, PRIMARYKEY_TIMESTAMP_COL_ID, tableUid, &s); + return columnListInsert(pColumnList, tableUid, &s, TSDB_COL_NORMAL); } void columnCopy(SColumn* pDest, const SColumn* pSrc); @@ -817,8 +827,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil void columnCopy(SColumn* pDest, const SColumn* pSrc) { destroyFilterInfo(&pDest->info.flist); - pDest->columnIndex = pSrc->columnIndex; - pDest->tableUid = pSrc->tableUid; + pDest->uid = pSrc->uid; pDest->info.flist.numOfFilters = pSrc->info.flist.numOfFilters; pDest->info.flist.filterInfo = tFilterInfoDup(pSrc->info.flist.filterInfo, pSrc->info.flist.numOfFilters); pDest->info.type = pSrc->info.type; @@ -844,7 +853,7 @@ void columnListCopy(SArray* dst, const SArray* src, uint64_t uid) { for (int32_t i = 0; i < num; ++i) { SColumn* pCol = taosArrayGetP(src, i); - if (pCol->tableUid == uid) { + if (pCol->uid == uid) { SColumn* p = columnClone(pCol); taosArrayPush(dst, &p); } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 21ab193157..98a800c3c6 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -55,7 +55,7 @@ SSchema* getTableTagSchema(const STableMeta* pTableMeta) { return getOneColumnSchema(pTableMeta, getTableInfo(pTableMeta).numOfColumns); } -static tExprNode* createUnaryFunctionExprNode(int32_t functionId, SSchema* pSchema, tExprNode* pColumnNode) { +static tExprNode* createFunctionExprNode(int32_t functionId, SSchema* pSchema, tExprNode* pColumnNode, int32_t numOfCols) { if (pColumnNode == NULL) { pColumnNode = calloc(1, sizeof(tExprNode)); pColumnNode->nodeType = TEXPR_COL_NODE; @@ -66,9 +66,10 @@ static tExprNode* createUnaryFunctionExprNode(int32_t functionId, SSchema* pSche } tExprNode* pNode = calloc(1, sizeof(tExprNode)); - pNode->nodeType = TEXPR_UNARYEXPR_NODE; - pNode->_node.functionId = functionId; - pNode->_node.pLeft = pColumnNode; + pNode->nodeType = TEXPR_FUNCTION_NODE; + pNode->_function.functionId = functionId; + pNode->_function.pChild = pColumnNode; + pNode->_function.num = numOfCols; return pNode; } @@ -92,48 +93,50 @@ SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SC return NULL; } + uint64_t uid = 0; + if (pTableMetaInfo->pTableMeta) { + uid = pTableMetaInfo->pTableMeta->uid; + } + SSqlExpr* p = &pExpr->base; + p->pColumns = calloc(1, sizeof(SColumn)); + p->numOfCols = 1; if (pParamExpr != NULL) { - pExpr->pExpr = createUnaryFunctionExprNode(functionId, NULL, pParamExpr); + pExpr->pExpr = createFunctionExprNode(functionId, NULL, pParamExpr, 1); +// pExpr->base.pColumns + // todo set the correct number of columns } else if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { assert(pParamExpr == NULL); SSchema* s = getTbnameColumnSchema(); - p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; - pExpr->pExpr = createUnaryFunctionExprNode(functionId, s, pParamExpr); + setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_TAG, s); + + pExpr->pExpr = createFunctionExprNode(functionId, s, pParamExpr, 1); } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX || functionId == FUNCTION_BLKINFO) { assert(pParamExpr == NULL); + setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_UDC, pResSchema); - p->colInfo.colId = pColIndex->columnIndex; SSchema s = createSchema(pResSchema->type, pResSchema->bytes, pColIndex->columnIndex, pResSchema->name); - pExpr->pExpr = createUnaryFunctionExprNode(functionId, &s, pParamExpr); + pExpr->pExpr = createFunctionExprNode(functionId, &s, pParamExpr, 1); } else { - int32_t len = tListLen(p->colInfo.name); if (TSDB_COL_IS_TAG(pColIndex->type)) { SSchema* pSchema = getTableTagSchema(pTableMetaInfo->pTableMeta); - p->colInfo.colId = pSchema[pColIndex->columnIndex].colId; - pExpr->pExpr = createUnaryFunctionExprNode(functionId, &pSchema[pColIndex->columnIndex], pParamExpr); - snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema[pColIndex->columnIndex].name); + setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_TAG, &pSchema[pColIndex->columnIndex]); + pExpr->pExpr = createFunctionExprNode(functionId, &pSchema[pColIndex->columnIndex], pParamExpr, 1); } else if (pTableMetaInfo->pTableMeta != NULL) { // in handling select database/version/server_status(), the pTableMeta is NULL SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); - p->colInfo.colId = pSchema->colId; - snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema->name); + setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_NORMAL, pSchema); - pExpr->pExpr = createUnaryFunctionExprNode(functionId, pSchema, pParamExpr); + pExpr->pExpr = createFunctionExprNode(functionId, pSchema, pParamExpr, 1); } } - p->colInfo.flag = pColIndex->type; - p->colInfo.colIndex = pColIndex->columnIndex; + p->pColumns->flag = pColIndex->type; p->interBytes = interSize; memcpy(&p->resSchema, pResSchema, sizeof(SSchema)); - if (pTableMetaInfo->pTableMeta) { - p->uid = pTableMetaInfo->pTableMeta->uid; - } - return pExpr; } @@ -152,10 +155,9 @@ void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int assert(pExprInfo != NULL); SSqlExpr* pse = &pExprInfo->base; - pExprInfo->pExpr->_node.functionId = functionId; + pExprInfo->pExpr->_function.functionId = functionId; + assert(0); - pse->colInfo.colIndex = srcColumnIndex; - pse->colInfo.colId = colId; pse->resSchema.type = resType; pse->resSchema.bytes = resSize; } @@ -198,7 +200,7 @@ void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t byt int32_t getExprFunctionId(SExprInfo *pExprInfo) { assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); - return pExprInfo->pExpr->_node.functionId; + return pExprInfo->pExpr->_function.functionId; } void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { @@ -225,8 +227,9 @@ int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deep size_t size = taosArrayGetSize(src); for (int32_t i = 0; i < size; ++i) { SExprInfo* pExpr = taosArrayGetP(src, i); + uint64_t exprUid = pExpr->base.pColumns->uid; - if (pExpr->base.uid == uid) { + if (exprUid == uid) { if (deepcopy) { SExprInfo* p1 = calloc(1, sizeof(SExprInfo)); assignExprInfo(p1, pExpr); @@ -300,7 +303,7 @@ SArray* extractFunctionIdList(SArray* pExprInfoList) { SArray* p = taosArrayInit(len, sizeof(int32_t)); for(int32_t i = 0; i < len; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pExprInfoList, i); - taosArrayPush(p, &pExprInfo->pExpr->_node.functionId); + taosArrayPush(p, &pExprInfo->pExpr->_function.functionId); } return p; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 9fb50e1006..73c6b900f0 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -50,7 +50,7 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { strcpy(pTableMetaInfo->aliasName, name->tname); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pTableMeta->tableType = TSDB_NORMAL_TABLE; - pTableMeta->tableInfo.numOfColumns = 4; + pTableMeta->tableInfo.numOfColumns = 6; pTableMeta->tableInfo.rowSize = 28; pTableMeta->uid = 110; @@ -61,12 +61,12 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { setSchema(&pSchema[1], TSDB_DATA_TYPE_INT, 4, "a", 1); setSchema(&pSchema[2], TSDB_DATA_TYPE_DOUBLE, 8, "b", 2); setSchema(&pSchema[3], TSDB_DATA_TYPE_DOUBLE, 8, "col", 3); - -} + setSchema(&pSchema[4], TSDB_DATA_TYPE_BINARY, 12, "c", 4); + setSchema(&pSchema[5], TSDB_DATA_TYPE_BINARY, 44, "d", 5); } -TEST(testCase, validateAST_test) { - SSqlInfo info1 = doGenerateAST("select a a1111, a+b + 22, tbname from `t.1abc` where tsexprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 3); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_INT); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1111"); - ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.a"); - ASSERT_EQ(p1->base.colInfo.colId, 1); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "a"); - - ASSERT_EQ(taosArrayGetSize(pExprList), 3); - - SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p2->base.uid, 0); - ASSERT_EQ(p2->base.numOfParams, 1); // it is the serialized binary string of expression. - ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p2->base.resSchema.name, "a+b + 22"); - -// ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a"); -// ASSERT_EQ(p1->base.colInfo.colId, 1); -// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p2->base.token, "a+b + 22"); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3); + if (valid) { + ASSERT_EQ(ret, 0); + } else { + ASSERT_NE(ret, 0); + } destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); destroySqlInfo(&info1); } - -TEST(testCase, function_Test) { - SSqlInfo info1 = doGenerateAST("select count(a) from `t.1abc`"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 1); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a)"); - ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.a"); - ASSERT_EQ(p1->base.colInfo.colId, 1); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "count(a)"); - ASSERT_EQ(p1->base.interBytes, 8); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); } -TEST(testCase, function_Test2) { - SSqlInfo info1 = doGenerateAST("select count(a) abc from `t.1abc`"); - ASSERT_EQ(info1.valid, true); +//TEST(testCase, validateAST_test) { +// SSqlInfo info1 = doGenerateAST("select a a1111, a+b + 22, tbname from `t.1abc` where tsexprList; +// ASSERT_EQ(taosArrayGetSize(pExprList), 3); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->info.colId, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_INT); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1111"); +// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); +// ASSERT_EQ(p1->base.pColumns->info.colId, 1); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "a"); +// +// ASSERT_EQ(taosArrayGetSize(pExprList), 3); +// +// SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); +// ASSERT_EQ(p2->base.pColumns->uid, 0); +// ASSERT_EQ(p2->base.numOfParams, 1); // it is the serialized binary string of expression. +// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p2->base.resSchema.name, "a+b + 22"); +// +//// ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a"); +//// ASSERT_EQ(p1->base.colInfo.colId, 1); +//// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p2->base.token, "a+b + 22"); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, function_Test) { +// SSqlInfo info1 = doGenerateAST("select count(a) from `t.1abc`"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// +// SArray* pExprList = pQueryInfo->exprList; +// ASSERT_EQ(taosArrayGetSize(pExprList), 1); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a)"); +// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); +// ASSERT_EQ(p1->base.pColumns->info.colId, 1); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "count(a)"); +// ASSERT_EQ(p1->base.interBytes, 8); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, function_Test2) { +// SSqlInfo info1 = doGenerateAST("select count(a) abc from `t.1abc`"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// +// SArray* pExprList = pQueryInfo->exprList; +// ASSERT_EQ(taosArrayGetSize(pExprList), 1); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "abc"); +// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); +// ASSERT_EQ(p1->base.pColumns->info.colId, 1); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "count(a)"); +// ASSERT_EQ(p1->base.interBytes, 8); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, function_Test3) { +// SSqlInfo info1 = doGenerateAST("select first(*) from `t.1abc`"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// +// SArray* pExprList = pQueryInfo->exprList; +// ASSERT_EQ(taosArrayGetSize(pExprList), 4); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_TIMESTAMP); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "first(ts)"); +// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.ts"); +// ASSERT_EQ(p1->base.pColumns->info.colId, 0); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "first(ts)"); +// ASSERT_EQ(p1->base.interBytes, 24); +// +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 4); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, function_Test4) { +// SSqlInfo info1 = doGenerateAST("select _block_dist() as a1 from `t.1abc`"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// +// SArray* pExprList = pQueryInfo->exprList; +// ASSERT_EQ(taosArrayGetSize(pExprList), 1); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 1); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BINARY); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); +//// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); +//// ASSERT_EQ(p1->base.colInfo.colId, 0); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "_block_dist()"); +// ASSERT_EQ(p1->base.interBytes, 0); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 1); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; +//TEST(testCase, function_Test5) { +// //todo select concat(concat(a, b), concat(b, a)) from `t.1abc`; +// +// SSqlInfo info1 = doGenerateAST("select sum(a) + avg(b) as a1 from `t.1abc`"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_EQ(ret, 0); +// +// SArray* pExprList = pQueryInfo->exprList; +// ASSERT_EQ(taosArrayGetSize(pExprList), 3); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.numOfCols, 2); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// +// ASSERT_EQ(p1->base.numOfParams, 1); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); +// +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "sum(a) + avg(b)"); +// ASSERT_EQ(p1->base.interBytes, 0); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 1); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "abc"); - ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.a"); - ASSERT_EQ(p1->base.colInfo.colId, 1); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "count(a)"); - ASSERT_EQ(p1->base.interBytes, 8); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} - -TEST(testCase, function_Test3) { - SSqlInfo info1 = doGenerateAST("select first(*) from `t.1abc`"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 4); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_TIMESTAMP); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "first(ts)"); - ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); - ASSERT_EQ(p1->base.colInfo.colId, 0); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "first(ts)"); - ASSERT_EQ(p1->base.interBytes, 24); - - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 4); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} - -TEST(testCase, function_Test4) { - SSqlInfo info1 = doGenerateAST("select _block_dist() as a1 from `t.1abc`"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 1); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); - ASSERT_EQ(p1->base.numOfParams, 1); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BINARY); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); -// ASSERT_EQ(p1->base.colInfo.colId, 0); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "_block_dist()"); - ASSERT_EQ(p1->base.interBytes, 0); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 1); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} - -TEST(testCase, function_Test5) { - SSqlInfo info1 = doGenerateAST("select sum(a) + avg(b) as a1 from `t.1abc`"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_EQ(ret, 0); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 3); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 0); - ASSERT_EQ(p1->base.numOfParams, 1); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); -// ASSERT_EQ(p1->base.colInfo.colId, 0); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "sum(a) + avg(b)"); - ASSERT_EQ(p1->base.interBytes, 0); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); +TEST(testCase, function_Test10) { +// sqlCheck("select c from `t.1abc`", true); +// sqlCheck("select length(c) from `t.1abc`", true); + sqlCheck("select sum(length(a+b)) from `t.1abc`", false); +// sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); } TEST(testCase, function_Test6) { @@ -382,15 +428,15 @@ TEST(testCase, function_Test6) { ASSERT_EQ(taosArrayGetSize(pExprList), 5); SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); + ASSERT_EQ(p1->base.pColumns->uid, 110); ASSERT_EQ(p1->base.numOfParams, 0); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); ASSERT_EQ(p1->base.interBytes, 16); ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_node.functionId, FUNCTION_SUM); + ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_SUM); ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); tExprNode* pParam = p1->pExpr->_node.pLeft; @@ -404,15 +450,15 @@ TEST(testCase, function_Test6) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p2->base.uid, 110); + ASSERT_EQ(p2->base.pColumns->uid, 110); ASSERT_EQ(p2->base.numOfParams, 0); ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); - ASSERT_EQ(p2->base.colInfo.flag, TSDB_COL_NORMAL); + ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_NORMAL); ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); ASSERT_EQ(p2->base.interBytes, 24); ASSERT_EQ(p2->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p2->pExpr->_node.functionId, FUNCTION_FIRST); + ASSERT_EQ(p2->pExpr->_function.functionId, FUNCTION_FIRST); ASSERT_TRUE(p2->pExpr->_node.pRight == NULL); destroyQueryInfo(pQueryInfo); @@ -449,15 +495,15 @@ TEST(testCase, function_Test7) { ASSERT_EQ(taosArrayGetSize(pExprList), 2); SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.uid, 110); + ASSERT_EQ(p1->base.pColumns->uid, 110); ASSERT_EQ(p1->base.numOfParams, 0); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a+b)"); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); ASSERT_STRCASEEQ(p1->base.token, "count(a+b)"); ASSERT_EQ(p1->base.interBytes, 8); ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_node.functionId, FUNCTION_COUNT); + ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_COUNT); ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); tExprNode* pParam = p1->pExpr->_node.pLeft; @@ -504,16 +550,16 @@ TEST(testCase, function_Test8) { ASSERT_EQ(taosArrayGetSize(pExprList), 2); SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p1->base.uid, 110); + ASSERT_EQ(p1->base.pColumns->uid, 110); ASSERT_EQ(p1->base.numOfParams, 1); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); ASSERT_EQ(p1->base.interBytes, 16); ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_node.functionId, FUNCTION_TOP); + ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); tExprNode* pParam = p1->pExpr->_node.pLeft; diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 528116fc32..c357d92b11 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -95,16 +95,16 @@ TEST(testCase, planner_test) { ASSERT_EQ(taosArrayGetSize(pExprList), 2); SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p1->base.uid, 110); + ASSERT_EQ(p1->base.pColumns->uid, 110); ASSERT_EQ(p1->base.numOfParams, 1); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); ASSERT_EQ(p1->base.interBytes, 16); - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_node.functionId, FUNCTION_TOP); + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); tExprNode* pParam = p1->pExpr->_node.pLeft; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 969b2c7622..25e63507c2 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -186,7 +186,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe for (int32_t i = 0; i < numOfCols; ++i) { SColumn* pCol = taosArrayGetP(tableCols, i); - SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex}; + SColumnIndex index = {.tableIndex = 0, /*.columnIndex = pCol->columnIndex*/}; SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, i); SSchema resultSchema = *pSchema; From 013f8fed42c75ccef4c480a16e6cb21fc5fd4134 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Nov 2021 10:35:50 +0800 Subject: [PATCH 06/26] [td-10564] 1. Enable and then refactor the parse the nested function in select clause. 2. Add some scalar string function in function module. 3. Add more test cases. --- include/libs/function/function.h | 2 +- include/libs/parser/parser.h | 6 +- include/util/tdef.h | 5 +- source/libs/executor/src/executorimpl.c | 64 +-- source/libs/function/inc/tscalarfunction.h | 2 +- source/libs/function/src/tscalarfunction.c | 62 ++- source/libs/parser/inc/queryInfoUtil.h | 2 +- source/libs/parser/src/astValidate.c | 306 ++++++++----- source/libs/parser/src/queryInfoUtil.c | 25 +- source/libs/parser/src/ttokenizer.c | 2 +- source/libs/parser/test/parserTests.cpp | 481 +++++++++++---------- source/libs/parser/test/plannerTest.cpp | 124 +++--- source/libs/planner/src/planner.c | 14 +- 13 files changed, 618 insertions(+), 477 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 1bc0128dbe..a8872ad323 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -221,7 +221,7 @@ typedef struct SScalarFunctionInfo { char name[FUNCTIONS_NAME_MAX_LENGTH]; int8_t type; // scalar function or aggregation function uint32_t functionId; // index of scalar function - void (*process)(const struct SScalarFuncParam *pInput, struct SScalarFuncParam* pOutput); + void (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput); } SScalarFunctionInfo; typedef struct SMultiFunctionsDesc { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index f79ad35862..7b8d50bf16 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -80,8 +80,7 @@ typedef struct SQueryStmtInfo { SGroupbyExpr groupbyExpr; // groupby tags info SArray * colList; // SArray SFieldInfo fieldsInfo; - SArray * exprList; // SArray - SArray * exprList1; // final exprlist in case of arithmetic expression exists + SArray** exprList; // SArray SLimit limit; SLimit slimit; STagCond tagCond; @@ -112,6 +111,7 @@ typedef struct SQueryStmtInfo { struct SQueryStmtInfo *pDownstream; int32_t havingFieldNum; SMultiFunctionsDesc info; + int32_t exprListLevelIndex; } SQueryStmtInfo; typedef struct SColumnIndex { @@ -165,7 +165,7 @@ void assignExprInfo(SExprInfo* dst, const SExprInfo* src); void columnListCopy(SArray* dst, const SArray* src, uint64_t uid); void columnListDestroy(SArray* pColumnList); -void dropAllExprInfo(SArray* pExprInfo); +void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel); SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, struct tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); diff --git a/include/util/tdef.h b/include/util/tdef.h index 652d329cd3..65230e2a7e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -140,8 +140,9 @@ do { \ #define FUNCTION_ROUND 4503 #define FUNCTION_LENGTH 4800 -#define FUNCTION_LTRIM 4801 -#define FUNCTION_RTRIM 4802 +#define FUNCTION_CONCAT 4801 +#define FUNCTION_LTRIM 4802 +#define FUNCTION_RTRIM 4803 #define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index aa7ea48432..7e19093b10 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,10 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include #include "os.h" -#include "ttime.h" #include "taosmsg.h" #include "tglobal.h" +#include "ttime.h" #include "exception.h" #include "executorimpl.h" @@ -161,7 +162,7 @@ int64_t genQueryId(void) { static int32_t getExprFunctionId(SExprInfo *pExprInfo) { assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); - return pExprInfo->pExpr->_node.functionId; + return pExprInfo->pExpr->_function.functionId; } static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { @@ -1069,6 +1070,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo } static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +#if 0 for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1079,7 +1081,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, if (pCtx[i].functionId == FUNCTION_ARITHM) { // setArithParams((SScalarFunctionSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); } else { - SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; + SColIndex* pCol = &pOperator->pExpr[i].base.pColumns->info.; if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || (pCtx[i].functionId == FUNCTION_BLKINFO) || (TSDB_COL_IS_TAG(pCol->flag) && pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; @@ -1087,7 +1089,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, // in case of the block distribution query, the inputBytes is not a constant value. pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); + assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); if (pCtx[i].functionId < 0) { SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); @@ -1112,7 +1114,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); pCtx[i].pInput = p->pData; - assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); + assert(p->info.colId == pColIndex->info.colId && pCtx[i].inputType == p->info.type); for(int32_t j = 0; j < pBlock->info.rows; ++j) { char* dst = p->pData + j * p->info.bytes; taosVariantDump(&pOperator->pExpr[i].base.param[1], dst, p->info.type, true); @@ -1120,6 +1122,8 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, } } } +#endif + } static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { @@ -1170,11 +1174,11 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, continue; } - SColIndex * pColIndex = &pExpr[k].base.colInfo; + SColIndex * pColIndex = NULL/*&pExpr[k].base.colInfo*/; int16_t index = pColIndex->colIndex; SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); - assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); +// assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { @@ -1858,7 +1862,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr for (int32_t i = 0; i < numOfOutput; ++i) { SSqlExpr *pSqlExpr = &pExpr[i].base; SQLFunctionCtx* pCtx = &pFuncCtx[i]; - +#if 0 SColIndex *pIndex = &pSqlExpr->colInfo; if (TSDB_COL_REQ_NULL(pIndex->flag)) { @@ -1867,7 +1871,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr } else { pCtx->requireNull = false; } - +#endif // pCtx->inputBytes = pSqlExpr->colBytes; // pCtx->inputType = pSqlExpr->colType; @@ -2359,7 +2363,7 @@ bool onlyQueryTags(SQueryAttr* pQueryAttr) { if (functionId != FUNCTION_TAGPRJ && functionId != FUNCTION_TID_TAG && - (!(functionId == FUNCTION_COUNT && pExprInfo->base.pColumns->colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == FUNCTION_COUNT && pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX)) && (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.pColumns->flag)))) { return false; } @@ -2879,7 +2883,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData int32_t numOfOutput = pTableScanInfo->numOfOutput; for (int32_t i = 0; i < numOfOutput; ++i) { int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.pColumns->colId; + int32_t colId = pTableScanInfo->pExpr[i].base.pColumns->info.colId; // group by + first/last should not apply the first/last block filter if (functionId < 0) { @@ -3211,7 +3215,7 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt } // todo use tag column index to optimize performance - doSetTagValueInParam(pTable, pLocalExprInfo->base.pColumns->colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type, + doSetTagValueInParam(pTable, pLocalExprInfo->base.pColumns->info.colId, &pCtx[idx].tag, pLocalExprInfo->base.resSchema.type, pLocalExprInfo->base.resSchema.bytes); if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resSchema.type) @@ -3787,7 +3791,7 @@ void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); for (int32_t k = 0; k < numOfCols; ++k) { SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->colId == pExpr->colInfo.colId) { + if (pres->info.colId == pExpr->colInfo.colId) { pCtx[i].param[0].arr = pres->pResult; break; } @@ -3819,7 +3823,7 @@ void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunction int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); for (int32_t k = 0; k < numOfCols; ++k) { SStddevInterResult* pres = taosArrayGet(p->pResult, k); - if (pres->colId == pExpr1->colInfo.colId) { + if (pres->info.colId == pExpr1->colInfo.colId) { pCtx[i].param[0].arr = pres->pResult; break; } @@ -5041,7 +5045,7 @@ SArray* getOrderCheckColumns(SQueryAttr* pQuery) { SSqlExpr* pExpr = &pQuery->pExpr1[j].base; int32_t functionId = getExprFunctionId(&pQuery->pExpr1[j]); - if (index->colId == pExpr->colInfo.colId && + if (index->colId == pExpr->pColumns->info.colId && (functionId == FUNCTION_PRJ || functionId == FUNCTION_TAG || functionId == FUNCTION_TS)) { index->colIndex = j; index->colId = pExpr->resSchema.colId; @@ -5073,8 +5077,8 @@ SArray* getResultGroupCheckColumns(SQueryAttr* pQuery) { // FUNCTION_TAG_DUMMY function needs to be ignored if (index->colId == pExpr->pColumns->info.colId && - ((TSDB_COL_IS_TAG(pExpr->colInfo.flag) && functionId == FUNCTION_TAG) || - (TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && functionId == FUNCTION_PRJ))) { + ((TSDB_COL_IS_TAG(pExpr->pColumns->flag) && functionId == FUNCTION_TAG) || + (TSDB_COL_IS_NORMAL_COL(pExpr->pColumns->flag) && functionId == FUNCTION_PRJ))) { index->colIndex = j; index->colId = pExpr->resSchema.colId; found = true; @@ -5291,7 +5295,7 @@ SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); for(int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData col = {{0}}; - col.info.colId = pExpr[i].base.pColumns->colId; + col.info.colId = pExpr[i].base.pColumns->info.colId; // col.info.bytes = pExpr[i].base.colBytes; // col.info.type = pExpr[i].base.colType; taosArrayPush(pDataBlock->pDataBlock, &col); @@ -6777,7 +6781,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { int16_t type = pExprInfo->base.resSchema.type; for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->colId) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { bytes = pQueryAttr->tagColList[i].bytes; type = pQueryAttr->tagColList[i].type; break; @@ -6809,10 +6813,10 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { output += sizeof(pQueryAttr->vgId); char* data = NULL; - if (pExprInfo->base.pColumns->colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->colId, type, bytes); + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); } doSetTagValueToResultBuf(output, data, type, bytes); @@ -6848,10 +6852,10 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { type = pExprInfo[j].base.resSchema.type; bytes = pExprInfo[j].base.resSchema.bytes; - if (pExprInfo[j].base.pColumns->colId == TSDB_TBNAME_COLUMN_INDEX) { + if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->colId, type, bytes); + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); } dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; @@ -7047,20 +7051,20 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *pExpr, SColumnInfo* pTagCols) { int32_t j = 0; - if (TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + if (TSDB_COL_IS_TAG(pExpr->pColumns->flag)) { + if (pExpr->pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { return TSDB_TBNAME_COLUMN_INDEX; } while(j < pTableInfo->numOfTags) { - if (pExpr->colInfo.colId == pTagCols[j].colId) { + if (pExpr->pColumns->info.colId == pTagCols[j].colId) { return j; } j += 1; } - } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data + } /*else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { // user specified column data return TSDB_UD_COLUMN_INDEX; } else { while (j < pTableInfo->numOfCols) { @@ -7070,7 +7074,7 @@ static int32_t getColumnIndexInSource(SQueriedTableInfo *pTableInfo, SSqlExpr *p j += 1; } - } + }*/ return INT32_MIN; // return a less than TSDB_TBNAME_COLUMN_INDEX value } @@ -7815,7 +7819,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu // bytes = tDataTypes[type].bytes; // } else { // int32_t index = pExprs[i].base.colInfo.colIndex; -// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.pColumns->colId); +// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.pColumns->info.colId); // // type = prevExpr[index].base.resSchema.type; // bytes = prevExpr[index].base.resSchema.bytes; @@ -7951,7 +7955,7 @@ static void doUpdateExprColumnIndex(SQueryAttr *pQueryAttr) { // } // todo opt performance - SColIndex *pColIndex = &pSqlExprMsg->colInfo; + SColIndex *pColIndex = NULL;/*&pSqlExprMsg->colInfo;*/ if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { int32_t f = 0; for (f = 0; f < pQueryAttr->numOfCols; ++f) { diff --git a/source/libs/function/inc/tscalarfunction.h b/source/libs/function/inc/tscalarfunction.h index ddb56518e0..6d23775610 100644 --- a/source/libs/function/inc/tscalarfunction.h +++ b/source/libs/function/inc/tscalarfunction.h @@ -37,7 +37,7 @@ typedef struct SScalarFunctionSupport { char** data; } SScalarFunctionSupport; -extern struct SScalarFunctionInfo scalarFunc[5]; +extern struct SScalarFunctionInfo scalarFunc[8]; int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncParam* pOutput, void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t)); diff --git a/source/libs/function/src/tscalarfunction.c b/source/libs/function/src/tscalarfunction.c index 80d426f101..1e7e3ef155 100644 --- a/source/libs/function/src/tscalarfunction.c +++ b/source/libs/function/src/tscalarfunction.c @@ -9,8 +9,9 @@ static void assignBasicParaInfo(struct SScalarFuncParam* dst, const struct SScal dst->num = src->num; } -static void tceil(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { +static void tceil(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); + assert(numOfInput == 1); switch (pLeft->bytes) { case TSDB_DATA_TYPE_FLOAT: { @@ -34,8 +35,9 @@ static void tceil(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { } } -static void tfloor(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { +static void tfloor(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); + assert(numOfInput == 1); switch (pLeft->bytes) { case TSDB_DATA_TYPE_FLOAT: { @@ -61,8 +63,9 @@ static void tfloor(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { } } -static void _tabs(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { +static void _tabs(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); + assert(numOfInput == 1); switch (pLeft->bytes) { case TSDB_DATA_TYPE_FLOAT: { @@ -118,8 +121,9 @@ static void _tabs(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { } } -static void tround(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { +static void tround(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { assignBasicParaInfo(pOutput, pLeft); + assert(numOfInput == 1); switch (pLeft->bytes) { case TSDB_DATA_TYPE_FLOAT: { @@ -143,7 +147,9 @@ static void tround(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { } } -static void tlength(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { +static void tlength(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { + assert(numOfInput == 1); + int64_t* out = (int64_t*) pOutput->data; char* s = pLeft->data; @@ -152,6 +158,46 @@ static void tlength(const SScalarFuncParam *pLeft, SScalarFuncParam* pOutput) { } } +static void tconcat(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { + assert(numOfInput > 0); + + int32_t rowLen = 0; + int32_t num = 1; + for(int32_t i = 0; i < numOfInput; ++i) { + rowLen += pLeft[i].bytes; + + if (pLeft[i].num > 1) { + num = pLeft[i].num; + } + } + + pOutput->data = realloc(pOutput->data, rowLen * num); + assert(pOutput->data); + + char* rstart = pOutput->data; + for(int32_t i = 0; i < num; ++i) { + + char* s = rstart; + varDataSetLen(s, 0); + for (int32_t j = 0; j < numOfInput; ++j) { + char* p1 = POINTER_SHIFT(pLeft[j].data, i * pLeft[j].bytes); + + memcpy(varDataVal(s) + varDataLen(s), varDataVal(p1), varDataLen(p1)); + varDataLen(s) += varDataLen(p1); + } + + rstart += rowLen; + } +} + +static void tltrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { + +} + +static void trtrim(SScalarFuncParam* pOutput, size_t numOfInput, const SScalarFuncParam *pLeft) { + +} + static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { case TSDB_DATA_TYPE_TINYINT: @@ -256,7 +302,6 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(pExprs->_node.optr); SScalarFuncParam left = {0}, right = {0}; - if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE || pLeft->nodeType == TEXPR_UNARYEXPR_NODE) { setScalarFuncParam(&left, leftOutput.type, leftOutput.bytes, leftOutput.data, leftOutput.num); } else if (pLeft->nodeType == TEXPR_COL_NODE) { @@ -318,12 +363,15 @@ int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarFuncPa return 0; } -SScalarFunctionInfo scalarFunc[5] = { +SScalarFunctionInfo scalarFunc[8] = { {"ceil", FUNCTION_TYPE_SCALAR, FUNCTION_CEIL, tceil}, {"floor", FUNCTION_TYPE_SCALAR, FUNCTION_FLOOR, tfloor}, {"abs", FUNCTION_TYPE_SCALAR, FUNCTION_ABS, _tabs}, {"round", FUNCTION_TYPE_SCALAR, FUNCTION_ROUND, tround}, {"length", FUNCTION_TYPE_SCALAR, FUNCTION_LENGTH, tlength}, + {"concat", FUNCTION_TYPE_SCALAR, FUNCTION_CONCAT, tconcat}, + {"ltrim", FUNCTION_TYPE_SCALAR, FUNCTION_LTRIM, tltrim}, + {"rtrim", FUNCTION_TYPE_SCALAR, FUNCTION_RTRIM, trtrim}, }; void setScalarFunctionSupp(struct SScalarFunctionSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 68fe08db47..6edcc4b225 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -31,7 +31,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo); SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema); -void addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo); +void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t level); void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index c25e647b48..d7e367cd0f 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -202,7 +202,6 @@ SQueryStmtInfo *createQueryInfo() { SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; pQueryInfo->limit.limit = -1; @@ -213,6 +212,13 @@ SQueryStmtInfo *createQueryInfo() { pQueryInfo->pUpstream = taosArrayInit(4, POINTER_BYTES); pQueryInfo->window = TSWINDOW_INITIALIZER; + pQueryInfo->exprList = calloc(10, POINTER_BYTES); + for(int32_t i = 0; i < 10; ++i) { + pQueryInfo->exprList[i] = taosArrayInit(4, POINTER_BYTES); + } + + pQueryInfo->exprListLevelIndex = 0; + return pQueryInfo; } @@ -221,14 +227,9 @@ static void destroyQueryInfoImpl(SQueryStmtInfo* pQueryInfo) { cleanupColumnCond(&pQueryInfo->colCond); cleanupFieldInfo(&pQueryInfo->fieldsInfo); - dropAllExprInfo(pQueryInfo->exprList); + dropAllExprInfo(pQueryInfo->exprList, 10); pQueryInfo->exprList = NULL; - if (pQueryInfo->exprList1 != NULL) { - dropAllExprInfo(pQueryInfo->exprList1); - pQueryInfo->exprList1 = NULL; - } - columnListDestroy(pQueryInfo->colList); pQueryInfo->colList = NULL; @@ -1707,18 +1708,27 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f } } +SArray* getCurrentExprList(SQueryStmtInfo* pQueryInfo) { + assert(pQueryInfo != NULL && pQueryInfo->exprListLevelIndex >= 0 && pQueryInfo->exprListLevelIndex < 10); + return pQueryInfo->exprList[pQueryInfo->exprListLevelIndex]; +} + SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, SSchema* pResultSchema, tExprNode* pExprNode, int32_t interSize, const char* token, bool finalResult) { STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); SExprInfo* pExpr = createExprInfo(pTableMetaInfo, functionId, pIndex, pExprNode, pResultSchema, interSize); - addExprInfo(pQueryInfo, outputColIndex, pExpr); + + SArray* pExprList = getCurrentExprList(pQueryInfo); + addExprInfo(pExprList, outputColIndex, pExpr, pQueryInfo->exprListLevelIndex); tstrncpy(pExpr->base.token, token, sizeof(pExpr->base.token)); uint64_t uid = pTableMetaInfo->pTableMeta->uid; - SArray* p = TSDB_COL_IS_TAG(pIndex->type)?pTableMetaInfo->tagColList:pQueryInfo->colList; - columnListInsert(p, uid, pColSchema, pIndex->type); + if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) { + SArray* p = TSDB_COL_IS_TAG(pIndex->type) ? pTableMetaInfo->tagColList : pQueryInfo->colList; + columnListInsert(p, uid, pColSchema, pIndex->type); + } pExpr->base.pColumns->flag = pIndex->type; if (TSDB_COL_IS_NORMAL_COL(pIndex->type)) { @@ -1792,7 +1802,8 @@ static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTab SExprInfo* pExpr = createExprInfo(pTableMetaInfo, FUNCTION_TS_DUMMY, &indexTS, NULL, &s, TSDB_KEYSIZE); strncpy(pExpr->base.token, "ts", tListLen(pExpr->base.token)); - addExprInfo(pQueryInfo, outputIndex, pExpr); + SArray* pExprList = getCurrentExprList(pQueryInfo); + addExprInfo(pExprList, outputIndex, pExpr, pQueryInfo->exprListLevelIndex); SSchema* pSourceSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, indexTS.columnIndex); columnListInsert(pQueryInfo->colList, pTableMetaInfo->pTableMeta->uid, pSourceSchema, TSDB_COL_NORMAL); @@ -1900,8 +1911,9 @@ static int32_t doHandleOneParam(SQueryStmtInfo *pQueryInfo, tSqlExprItem* pItem, static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumnList, SMsgBuf* pMsgBuf); static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf); -int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId, STableMetaInfo** pTableMetaInfo, SSchema* columnSchema, - tExprNode** pNode, SColumnIndex* pIndex, tSqlExprItem* pParamElem, SMsgBuf* pMsgBuf) { +int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId, STableMetaInfo** pTableMetaInfo, + SSchema* columnSchema, tExprNode** pNode, SColumnIndex* pIndex, + tSqlExprItem* pParamElem, SMsgBuf* pMsgBuf) { const char* msg1 = "not support column types"; const char* msg2 = "invalid parameters"; const char* msg3 = "illegal column name"; @@ -1926,11 +1938,20 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId return buildInvalidOperationMsg(pMsgBuf, msg6); } - int32_t code = createComplexExpr(pQueryInfo, i, pParamElem, pMsgBuf); + SArray* pExprList = getCurrentExprList(pQueryInfo); + size_t n = taosArrayGetSize(pExprList); + + // todo extract the table uid + pIndex->tableIndex = 0; + int32_t code = createComplexExpr(pQueryInfo, n, pParamElem, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } + SExprInfo** pLastExpr = taosArrayGetLast(pExprList); + *pNode = (*pLastExpr)->pExpr; + *(SSchema*) columnSchema = (*pLastExpr)->base.resSchema; + *pTableMetaInfo = getMetaInfo(pQueryInfo, 0); } else { if ((getColumnIndexByName(&pParamElem->pNode->columnName, pQueryInfo, pIndex, pMsgBuf) != TSDB_CODE_SUCCESS)) { return buildInvalidOperationMsg(pMsgBuf, msg3); @@ -1945,36 +1966,20 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId *pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); *columnSchema = *(SSchema*)getOneColumnSchema((*pTableMetaInfo)->pTableMeta, pIndex->columnIndex); } - }else if (tokenId == TK_PLUS || tokenId == TK_MINUS || tokenId == TK_STAR || tokenId == TK_REM || tokenId == TK_DIVIDE || tokenId == TK_CONCAT) { - int32_t arithmeticType = NON_ARITHMEIC_EXPR; - SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); - if (validateComplexExpr(pParamElem->pNode, pQueryInfo, pColumnList, &arithmeticType, pMsgBuf) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + } else if (tokenId == TK_PLUS || tokenId == TK_MINUS || tokenId == TK_STAR || tokenId == TK_REM || tokenId == TK_DIVIDE || tokenId == TK_CONCAT) { + pIndex->tableIndex = 0; // todo set the correct table index + + SArray* pExprList = getCurrentExprList(pQueryInfo); + size_t n = taosArrayGetSize(pExprList); + int32_t code = createComplexExpr(pQueryInfo, n, pParamElem, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; } - if (arithmeticType != NORMAL_ARITHMETIC) { - return buildInvalidOperationMsg(pMsgBuf, msg4); - } - - *pTableMetaInfo = getMetaInfo(pQueryInfo, 0); // todo get the first table meta. - *columnSchema = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); - - SToken* pExprToken = &pParamElem->pNode->exprToken; - int32_t len = MIN(TSDB_COL_NAME_LEN, pExprToken->n + 1); - tstrncpy(columnSchema->name, pExprToken->z, len); - - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = sqlExprToExprNode(pNode, pParamElem->pNode, pQueryInfo, colList, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(colList); - tExprTreeDestroy(*pNode, NULL); - return buildInvalidOperationMsg(pMsgBuf, msg2); - } - - pIndex->tableIndex = 0; - multiColumnListInsert(pQueryInfo, pColumnList, pMsgBuf); - taosArrayDestroy(colList); - taosArrayDestroy(pColumnList); + SExprInfo** pLastExpr = taosArrayGetLast(getCurrentExprList(pQueryInfo)); + *pNode = (*pLastExpr)->pExpr; + *(SSchema*) columnSchema = (*pLastExpr)->base.resSchema; + *pTableMetaInfo = getMetaInfo(pQueryInfo, 0); } else { assert(0); } @@ -1997,7 +2002,7 @@ static int32_t checkForkParam(tSqlExpr* pSqlExpr, size_t k, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } -int32_t addExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult, SMsgBuf* pMsgBuf) { +int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult, SMsgBuf* pMsgBuf) { STableMetaInfo* pTableMetaInfo = NULL; int32_t functionId = pItem->functionId; int32_t code = TSDB_CODE_SUCCESS; @@ -2071,12 +2076,15 @@ int32_t addExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSqlEx tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->Expr.paramList, 0); - tExprNode* pNode = NULL; - int32_t tokenId = pParamElem->pNode->tokenId; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; + tExprNode* pNode = NULL; + int32_t tokenId = pParamElem->pNode->tokenId; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; SSchema columnSchema = {0}; - code = extractFunctionParameterInfo(pQueryInfo, tokenId, &pTableMetaInfo, &columnSchema, &pNode, &index, pParamElem,pMsgBuf); + pQueryInfo->exprListLevelIndex += 1; + code = extractFunctionParameterInfo(pQueryInfo, tokenId, &pTableMetaInfo, &columnSchema, &pNode, &index, pParamElem, pMsgBuf); + pQueryInfo->exprListLevelIndex -= 1; + if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2440,23 +2448,31 @@ static int32_t validateExprLeafFunctionNode(SQueryStmtInfo* pQueryInfo, tSqlExpr } int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); - if (addExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) - int32_t inc = (int32_t) getNumOfExprs(pQueryInfo) - outputIndex; - if (inc > 1) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - // Not supported data type in expression - for(int32_t i = 0; i < inc; ++i) { - SExprInfo* p1 = getExprInfo(pQueryInfo, i + outputIndex); - int16_t t = p1->base.resSchema.type; - if (t == TSDB_DATA_TYPE_TIMESTAMP) { + if (scalar) { + printf("scalar function found!\n"); +// if (createComplexExpr(pQueryInfo, outputIndex, &item, pMsgBuf) != TSDB_CODE_SUCCESS) { +// return TSDB_CODE_TSC_INVALID_OPERATION; +// } + } else { + if (addAggExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + + // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) + int32_t inc = (int32_t)getNumOfExprs(pQueryInfo) - outputIndex; + if (inc > 1) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // Not supported data type in expression + for (int32_t i = 0; i < inc; ++i) { + SExprInfo* p1 = getExprInfo(pQueryInfo, i + outputIndex); + int16_t t = p1->base.resSchema.type; + if (t == TSDB_DATA_TYPE_TIMESTAMP) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } } return TSDB_CODE_SUCCESS; @@ -2480,7 +2496,6 @@ int32_t validateScalarFunctionParamNum(tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { int32_t validateScalarFunctionParam(SQueryStmtInfo* pQueryInfo, tSqlExpr* pExpr, SArray* pList, int32_t* exprType, SMsgBuf* pMsgBuf) { int32_t code = TSDB_CODE_SUCCESS; - // more than one parameter for count() function SArray* pParamList = pExpr->Expr.paramList; *exprType = NORMAL_ARITHMETIC; @@ -2491,6 +2506,7 @@ int32_t validateScalarFunctionParam(SQueryStmtInfo* pQueryInfo, tSqlExpr* pExpr, int32_t type = pSqlExpr->type; if (type == SQL_NODE_VALUE) { + // do nothing for scalar function, or maybe the evaluation can be done here } else if (type == SQL_NODE_SQLFUNCTION) { code = validateExprLeafFunctionNode(pQueryInfo, pSqlExpr, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { @@ -2785,45 +2801,95 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt tExprNode* pLeft = NULL; tExprNode* pRight= NULL; - if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { - // assert it is a scalar function - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; - (*pExpr)->_function.num = 1; - (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); - - SArray* pParamList = pSqlExpr->Expr.paramList; - size_t num = taosArrayGetSize(pParamList); - (*pExpr)->_function.pChild = calloc(num, POINTER_BYTES); - - for(int32_t i = 0; i < num; ++i) { - tSqlExprItem* pItem = taosArrayGet(pParamList, i); - sqlExprToExprNode(&(((*pExpr)->_function.pChild)[0]), pItem->pNode, pQueryInfo, pCols, pMsgBuf); - } - - return TSDB_CODE_SUCCESS; - } +// if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { +// // assert it is a scalar function +// *pExpr = calloc(1, sizeof(tExprNode)); +// (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; +// (*pExpr)->_function.num = 1; +// (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); +// +// SArray* pParamList = pSqlExpr->Expr.paramList; +// size_t num = taosArrayGetSize(pParamList); +// (*pExpr)->_function.pChild = calloc(num, POINTER_BYTES); +// +// for(int32_t i = 0; i < num; ++i) { +// tSqlExprItem* pItem = taosArrayGet(pParamList, i); +// sqlExprToExprNode(&(((*pExpr)->_function.pChild)[0]), pItem->pNode, pQueryInfo, pCols, pMsgBuf); +// } +// +// return TSDB_CODE_SUCCESS; +// } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (pSqlExpr->pLeft != NULL) { - int32_t ret = sqlExprToExprNode(&pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if (pSqlExpr->type == SQL_NODE_EXPR) { + if (pSqlExpr->pLeft != NULL) { + int32_t ret = sqlExprToExprNode(&pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } } - } - if (pSqlExpr->pRight != NULL) { - int32_t ret = sqlExprToExprNode(&pRight, pSqlExpr->pRight, pQueryInfo, pCols, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pLeft, NULL); - return ret; + if (pSqlExpr->pRight != NULL) { + int32_t ret = sqlExprToExprNode(&pRight, pSqlExpr->pRight, pQueryInfo, pCols, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(pLeft, NULL); + return ret; + } } - } - if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->tokenId == 0) { - *pExpr = calloc(1, sizeof(tExprNode)); - return TSDB_CODE_SUCCESS; - } + if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->tokenId == 0) { + *pExpr = calloc(1, sizeof(tExprNode)); + return TSDB_CODE_SUCCESS; + } + } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { + SArray* pParamList = pSqlExpr->Expr.paramList; + + if (pParamList != NULL && taosArrayGetSize(pParamList) > 0) { + size_t num = taosArrayGetSize(pParamList); + + tExprNode** p = calloc(num, POINTER_BYTES); + pQueryInfo->exprListLevelIndex += 1; + + for(int32_t i = 0; i < num; ++i) { + tSqlExprItem* pItem = taosArrayGet(pParamList, i); + int32_t code = sqlExprToExprNode(&p[i], pItem->pNode, pQueryInfo, pCols, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pQueryInfo->exprListLevelIndex -= 1; + + bool scalar = false; + int32_t functionId = qIsBuiltinFunction(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n, &scalar); + if (functionId < 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); + + if (scalar) { + printf("scalar function found! %s\n", pSqlExpr->exprToken.z); + + // Expression on the results of aggregation functions + *pExpr = calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; + + (*pExpr)->_function.pChild = p; + (*pExpr)->_function.functionId = functionId; + (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n + 1); + return TSDB_CODE_SUCCESS; + } else { + printf("agg function found, %s\n", pSqlExpr->exprToken.z); + tSqlExprItem item = {.pNode = (tSqlExpr*)pSqlExpr, .aliasName = NULL, .functionId = functionId}; + if (addAggExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // convert the aggregate function to be the input data columns for the outer function. + } + } + } if (pSqlExpr->pLeft == NULL) { // it is the leaf node assert(pSqlExpr->pRight == NULL); @@ -2856,12 +2922,13 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt strncpy((*pExpr)->pSchema->name, pSqlExpr->exprToken.z, pSqlExpr->exprToken.n); // set the input column data byte and type. - size_t size = taosArrayGetSize(pQueryInfo->exprList); + SArray* pExprList = getCurrentExprList(pQueryInfo); + size_t size = taosArrayGetSize(pExprList); bool found = false; uint64_t uid = 0; for (int32_t i = 0; i < size; ++i) { - SExprInfo* p1 = taosArrayGetP(pQueryInfo->exprList, i); + SExprInfo* p1 = taosArrayGetP(pExprList, i); if (strcmp((*pExpr)->pSchema->name, p1->base.resSchema.name) == 0) { memcpy((*pExpr)->pSchema, &p1->base.resSchema, sizeof(SSchema)); @@ -2982,9 +3049,9 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, int32_t arithmeticType = NON_ARITHMEIC_EXPR; SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); - if (validateComplexExpr(pItem->pNode, pQueryInfo, pColumnList, &arithmeticType, pMsgBuf) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } +// if (validateComplexExpr(pItem->pNode, pQueryInfo, pColumnList, &arithmeticType, pMsgBuf) != TSDB_CODE_SUCCESS) { +// return buildInvalidOperationMsg(pMsgBuf, msg1); +// } if (arithmeticType == NORMAL_ARITHMETIC) { // expr string is set as the parameter of function @@ -2999,8 +3066,9 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, return buildInvalidOperationMsg(pMsgBuf, msg2); } - SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); - addExprInfo(pQueryInfo, exprIndex, pExpr); + SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); + SArray* pExprList = getCurrentExprList(pQueryInfo); + addExprInfo(pExprList, exprIndex, pExpr, pQueryInfo->exprListLevelIndex); setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); // check for if there is a tag in the arithmetic express @@ -3026,7 +3094,11 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, // set the serialized binary string as the parameter of arithmetic expression addExprInfoParam(&pExpr->base, c, TSDB_DATA_TYPE_BINARY, (int32_t)len); - addResColumnInfo(pQueryInfo, exprIndex, &pExpr->base.resSchema, pExpr); + + // Need to be added to the final result list. + if (pQueryInfo->exprListLevelIndex == 0) { + addResColumnInfo(pQueryInfo, exprIndex, &pExpr->base.resSchema, pExpr); + } tbufCloseWriter(&bw); taosArrayDestroy(colList); @@ -3043,8 +3115,9 @@ static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); } - SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); - addExprInfo(pQueryInfo, exprIndex, pExpr); + SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); + SArray* pExprList = getCurrentExprList(pQueryInfo); + addExprInfo(pExprList, exprIndex, pExpr, pQueryInfo->exprListLevelIndex); setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); @@ -3099,7 +3172,7 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, size_t numOfExpr = taosArrayGetSize(pSelNodeList); for (int32_t i = 0; i < numOfExpr; ++i) { - int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); + int32_t outputIndex = (int32_t) getNumOfExprs(pQueryInfo); tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i); int32_t type = pItem->pNode->type; @@ -3111,9 +3184,8 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, pQueryInfo->info.distinct = true; } - bool scalarFunc = false; - if (type == SQL_NODE_SQLFUNCTION) { + bool scalarFunc = false; pItem->functionId = qIsBuiltinFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n, &scalarFunc); if (pItem->functionId == FUNCTION_INVALID_ID) { int32_t functionId = FUNCTION_INVALID_ID; @@ -3123,15 +3195,17 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, } pItem->functionId = functionId; - } else if (scalarFunc) { - if ((code = createComplexExpr(pQueryInfo, i, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { - return code; - } } - // sql function in selection clause, append sql function info in pSqlCmd structure sequentially - if ((code = addExprAndResColumn(pQueryInfo, outputIndex, pItem, true, pMsgBuf)) != TSDB_CODE_SUCCESS) { - return code; + if (scalarFunc) { // scalar function + if ((code = createComplexExpr(pQueryInfo, outputIndex, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } + } else { // aggregate function + // sql function in selection clause, append sql function info in pSqlCmd structure sequentially + if ((code = addAggExprAndResColumn(pQueryInfo, outputIndex, pItem, true, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } } } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not @@ -3654,7 +3728,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer validateSqlNode(p, pQueryInfo, &buf); } - SArray* functionList = extractFunctionIdList(pQueryInfo->exprList); + SArray* functionList = extractFunctionIdList(pQueryInfo->exprList[0]); extractFunctionDesc(functionList, &pQueryInfo->info); if ((code = checkForInvalidExpr(pQueryInfo, &buf)) != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 98a800c3c6..df58651179 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -19,7 +19,7 @@ SSchema* getTbnameColumnSchema() { } size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo) { - return taosArrayGetSize(pQueryInfo->exprList); + return taosArrayGetSize(pQueryInfo->exprList[0]); } SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { @@ -104,7 +104,6 @@ SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SC if (pParamExpr != NULL) { pExpr->pExpr = createFunctionExprNode(functionId, NULL, pParamExpr, 1); -// pExpr->base.pColumns // todo set the correct number of columns } else if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { assert(pParamExpr == NULL); @@ -140,15 +139,17 @@ SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SC return pExpr; } -void addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo) { - assert(pQueryInfo != NULL && pQueryInfo->exprList != NULL); +void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t level) { + assert(pExprList != NULL ); - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList); + int32_t num = (int32_t) taosArrayGetSize(pExprList); if (index == num) { - taosArrayPush(pQueryInfo->exprList, &pExprInfo); + taosArrayPush(pExprList, &pExprInfo); } else { - taosArrayInsert(pQueryInfo->exprList, index, &pExprInfo); + taosArrayInsert(pExprList, index, &pExprInfo); } + + printf("add function, id:%d, level:%d\n", pExprInfo->pExpr->_function.functionId, level); } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { @@ -176,10 +177,10 @@ void destroyExprInfo(SExprInfo* pExprInfo) { tfree(pExprInfo); } -void dropAllExprInfo(SArray* pExprInfo) { +static void dropOneLevelExprInfo(SArray* pExprInfo) { size_t size = taosArrayGetSize(pExprInfo); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); destroyExprInfo(pExpr); } @@ -187,6 +188,12 @@ void dropAllExprInfo(SArray* pExprInfo) { taosArrayDestroy(pExprInfo); } +void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel) { + for(int32_t i = 0; i < numOfLevel; ++i) { + dropOneLevelExprInfo(pExprInfo[i]); + } +} + void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) { assert (pExpr != NULL || argument != NULL || bytes != 0); diff --git a/source/libs/parser/src/ttokenizer.c b/source/libs/parser/src/ttokenizer.c index b71fb4538e..683fe142ef 100644 --- a/source/libs/parser/src/ttokenizer.c +++ b/source/libs/parser/src/ttokenizer.c @@ -70,7 +70,7 @@ static SKeyword keywordTable[] = { {"STAR", TK_STAR}, {"SLASH", TK_SLASH}, {"REM ", TK_REM}, - {"CONCAT", TK_CONCAT}, + {"||", TK_CONCAT}, {"UMINUS", TK_UMINUS}, {"UPLUS", TK_UPLUS}, {"BITNOT", TK_BITNOT}, diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 73c6b900f0..24b62d1313 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -46,7 +46,7 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { SName* name = (SName*)taosArrayGet(req->pTableName, 0); memcpy(&pTableMetaInfo->name, taosArrayGet(req->pTableName, 0), sizeof(SName)); - pTableMetaInfo->pTableMeta = (STableMeta*)calloc(1, sizeof(STableMeta) + 4 * sizeof(SSchema)); + pTableMetaInfo->pTableMeta = (STableMeta*)calloc(1, sizeof(STableMeta) + 6 * sizeof(SSchema)); strcpy(pTableMetaInfo->aliasName, name->tname); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pTableMeta->tableType = TSDB_NORMAL_TABLE; @@ -99,6 +99,7 @@ void sqlCheck(const char* sql, bool valid) { qParserClearupMetaRequestInfo(&req); destroySqlInfo(&info1); } + } //TEST(testCase, validateAST_test) { @@ -125,7 +126,7 @@ void sqlCheck(const char* sql, bool valid) { // SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // -// SArray* pExprList = pQueryInfo->exprList; +// SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 3); // // SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); @@ -183,7 +184,7 @@ void sqlCheck(const char* sql, bool valid) { // SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // -// SArray* pExprList = pQueryInfo->exprList; +// SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 1); // // SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); @@ -229,7 +230,7 @@ void sqlCheck(const char* sql, bool valid) { // SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // -// SArray* pExprList = pQueryInfo->exprList; +// SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 1); // // SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); @@ -275,7 +276,7 @@ void sqlCheck(const char* sql, bool valid) { // SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // -// SArray* pExprList = pQueryInfo->exprList; +// SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 4); // // SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); @@ -320,7 +321,7 @@ void sqlCheck(const char* sql, bool valid) { // SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // -// SArray* pExprList = pQueryInfo->exprList; +// SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 1); // // SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); @@ -369,7 +370,7 @@ void sqlCheck(const char* sql, bool valid) { // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // ASSERT_EQ(ret, 0); // -// SArray* pExprList = pQueryInfo->exprList; +// SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 3); // // SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); @@ -395,236 +396,242 @@ void sqlCheck(const char* sql, bool valid) { TEST(testCase, function_Test10) { // sqlCheck("select c from `t.1abc`", true); // sqlCheck("select length(c) from `t.1abc`", true); - sqlCheck("select sum(length(a+b)) from `t.1abc`", false); +// sqlCheck("select sum(length(a+b)) from `t.1abc`", true); // sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); +// sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); +// sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); +// sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); + sqlCheck("select concat(a,b) from `t.1abc`", true); +// sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); +// sqlCheck("select length(length(length(a))) from `t.1abc`", true); } -TEST(testCase, function_Test6) { - SSqlInfo info1 = doGenerateAST("select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_EQ(ret, 0); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 5); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.pColumns->uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); - ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); - ASSERT_EQ(p1->base.interBytes, 16); - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_SUM); - ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); - - tExprNode* pParam = p1->pExpr->_node.pLeft; - - ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_ADD); - ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_COL_NODE); - ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_COL_NODE); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); - - SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p2->base.pColumns->uid, 110); - ASSERT_EQ(p2->base.numOfParams, 0); - ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); - ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); - ASSERT_EQ(p2->base.interBytes, 24); - ASSERT_EQ(p2->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p2->pExpr->_function.functionId, FUNCTION_FIRST); - ASSERT_TRUE(p2->pExpr->_node.pRight == NULL); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} - -TEST(testCase, function_Test7) { - SSqlInfo info1 = doGenerateAST("select count(a+b),count(1) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_EQ(ret, 0); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 2); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); - ASSERT_EQ(p1->base.pColumns->uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a+b)"); - ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "count(a+b)"); - ASSERT_EQ(p1->base.interBytes, 8); - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_COUNT); - ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); - - tExprNode* pParam = p1->pExpr->_node.pLeft; - - ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_ADD); - ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_COL_NODE); - ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_COL_NODE); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} - -TEST(testCase, function_Test8) { - SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_EQ(ret, 0); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 2); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p1->base.pColumns->uid, 110); - ASSERT_EQ(p1->base.numOfParams, 1); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.interBytes, 16); - - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); - ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); - ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); - - tExprNode* pParam = p1->pExpr->_node.pLeft; - - ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); - ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} - -TEST(testCase, invalid_sql_Test) { - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlInfo info1 = doGenerateAST("select count(k) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_NE(ret, 0); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -//=============================================================================================================== - info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - - pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_NE(ret, 0); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} \ No newline at end of file +//TEST(testCase, function_Test6) { +// SSqlInfo info1 = doGenerateAST("select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_EQ(ret, 0); +// +// SArray* pExprList = pQueryInfo->exprList[0]; +// ASSERT_EQ(taosArrayGetSize(pExprList), 5); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); +// ASSERT_EQ(p1->base.interBytes, 16); +// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); +// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_SUM); +// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); +// +// tExprNode* pParam = p1->pExpr->_node.pLeft; +// +// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); +// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_ADD); +// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_COL_NODE); +// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_COL_NODE); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); +// +// SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); +// ASSERT_EQ(p2->base.pColumns->uid, 110); +// ASSERT_EQ(p2->base.numOfParams, 0); +// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); +// ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); +// ASSERT_EQ(p2->base.interBytes, 24); +// ASSERT_EQ(p2->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); +// ASSERT_EQ(p2->pExpr->_function.functionId, FUNCTION_FIRST); +// ASSERT_TRUE(p2->pExpr->_node.pRight == NULL); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, function_Test7) { +// SSqlInfo info1 = doGenerateAST("select count(a+b),count(1) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_EQ(ret, 0); +// +// SArray* pExprList = pQueryInfo->exprList[0]; +// ASSERT_EQ(taosArrayGetSize(pExprList), 2); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a+b)"); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "count(a+b)"); +// ASSERT_EQ(p1->base.interBytes, 8); +// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); +// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_COUNT); +// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); +// +// tExprNode* pParam = p1->pExpr->_node.pLeft; +// +// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); +// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_ADD); +// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_COL_NODE); +// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_COL_NODE); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, function_Test8) { +// SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_EQ(ret, 0); +// +// SArray* pExprList = pQueryInfo->exprList[0]; +// ASSERT_EQ(taosArrayGetSize(pExprList), 2); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 1); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); +// ASSERT_EQ(p1->base.interBytes, 16); +// +// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); +// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); +// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); +// +// tExprNode* pParam = p1->pExpr->_node.pLeft; +// +// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); +// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); +// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); +// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} +// +//TEST(testCase, invalid_sql_Test) { +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlInfo info1 = doGenerateAST("select count(k) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_NE(ret, 0); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +////=============================================================================================================== +// info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_NE(ret, 0); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} \ No newline at end of file diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index c357d92b11..6e091bf8f4 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -66,65 +66,65 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { } } -TEST(testCase, planner_test) { - SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); - ASSERT_EQ(info1.valid, true); - - char msg[128] = {0}; - SMsgBuf buf; - buf.len = 128; - buf.buf = msg; - - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); - ASSERT_EQ(code, 0); - - SMetaReq req = {0}; - int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); - ASSERT_EQ(ret, 0); - ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); - - SQueryStmtInfo* pQueryInfo = createQueryInfo(); - setTableMetaInfo(pQueryInfo, &req); - - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); - ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); - ASSERT_EQ(ret, 0); - - SArray* pExprList = pQueryInfo->exprList; - ASSERT_EQ(taosArrayGetSize(pExprList), 2); - - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p1->base.pColumns->uid, 110); - ASSERT_EQ(p1->base.numOfParams, 1); - ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); - ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); - ASSERT_EQ(p1->base.interBytes, 16); - - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); - ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); - ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); - - tExprNode* pParam = p1->pExpr->_node.pLeft; - - ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); - ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); - ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); - - ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); - - struct SQueryPlanNode* n = nullptr; - code = qCreateQueryPlan(pQueryInfo, &n); - - char* str = NULL; - qQueryPlanToString(n, &str); - printf("%s\n", str); - - destroyQueryInfo(pQueryInfo); - qParserClearupMetaRequestInfo(&req); - destroySqlInfo(&info1); -} \ No newline at end of file +//TEST(testCase, planner_test) { +// SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_EQ(ret, 0); +// +// SArray* pExprList = pQueryInfo->exprList[0]; +// ASSERT_EQ(taosArrayGetSize(pExprList), 2); +// +// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 1); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); +// ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); +// ASSERT_EQ(p1->base.interBytes, 16); +// +// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); +// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); +// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); +// +// tExprNode* pParam = p1->pExpr->_node.pLeft; +// +// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); +// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); +// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); +// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); +// +// struct SQueryPlanNode* n = nullptr; +// code = qCreateQueryPlan(pQueryInfo, &n); +// +// char* str = NULL; +// qQueryPlanToString(n, &str); +// printf("%s\n", str); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} \ No newline at end of file diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 25e63507c2..53631fc9f1 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -226,9 +226,9 @@ static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryIn } if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->info.arithmeticOnAgg) { - int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); - pNode = - createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); +// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); +// pNode = +// createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); } if (pQueryInfo->fillType != TSDB_FILL_NONE) { @@ -292,7 +292,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { SArray* exprList = taosArrayInit(4, POINTER_BYTES); if (copyExprInfoList(exprList, pQueryInfo->exprList, uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - dropAllExprInfo(exprList); +// dropAllExprInfo(exprList); exit(-1); } @@ -308,7 +308,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { // 4. add the projection query node SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); columnListDestroy(tableColumnList); - dropAllExprInfo(exprList); +// dropAllExprInfo(exprList); taosArrayPush(upstream, &pNode); } @@ -316,7 +316,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { SQueryTableInfo info = {0}; int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList); SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList->pData, num, &info, NULL); + pQueryInfo->exprList[0]->pData, num, &info, NULL); // 4. add the aggregation or projection execution node pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList); @@ -338,7 +338,7 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { tfree(pQueryNode->info.name); tfree(pQueryNode->tableInfo.tableName); - dropAllExprInfo(pQueryNode->pExpr); +// dropAllExprInfo(pQueryNode->pExpr); if (pQueryNode->pPrevNodes != NULL) { int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes); From 6d2ecea1b097a2b92e3424012afc77055f3d5d15 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 Nov 2021 10:39:12 +0800 Subject: [PATCH 07/26] Merge branch '3.0' into feature/3.0_liaohj From f80294fc3d8c03d62bdb1ac1a9d66383fccd076d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 7 Nov 2021 15:49:30 +0800 Subject: [PATCH 08/26] [td-10564] Support two columns in the aggregate function. --- include/libs/function/function.h | 2 + include/libs/parser/parser.h | 11 +- source/libs/function/inc/taggfunction.h | 2 +- source/libs/function/src/taggfunction.c | 18 +- source/libs/parser/inc/parserUtil.h | 3 + source/libs/parser/inc/queryInfoUtil.h | 3 +- source/libs/parser/src/astValidate.c | 250 +++++++++++++++++------- source/libs/parser/src/parserUtil.c | 38 ++++ source/libs/parser/src/queryInfoUtil.c | 85 ++++---- source/libs/parser/test/parserTests.cpp | 16 +- source/libs/planner/src/planner.c | 2 +- 11 files changed, 301 insertions(+), 129 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index a8872ad323..8599b2a6f8 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -78,6 +78,8 @@ extern "C" { #define FUNCTION_MODE 36 #define FUNCTION_SAMPLE 37 +#define FUNCTION_COV 38 + // determine the real data need to calculated the result enum { BLK_DATA_NO_NEEDED = 0x0, diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 7b8d50bf16..5457f0b957 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -166,7 +166,16 @@ void columnListCopy(SArray* dst, const SArray* src, uint64_t uid); void columnListDestroy(SArray* pColumnList); void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel); -SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, struct tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize); + +typedef struct SSourceParam { +// struct tExprNode **pExprNode; + SArray *pExprNodeList; //Array +// SColumn *pCols; + SArray *pColumnList; //Array + int32_t num; +} SSourceParam; + +SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c3bc63cf6f..41c7309a18 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -28,7 +28,7 @@ extern "C" { #include "function.h" #include "tudf.h" -extern SAggFunctionInfo aggFunc[34]; +extern SAggFunctionInfo aggFunc[35]; typedef struct SResultRowEntryInfo { int8_t hasResult; // result generated, not NULL value diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index a2e6c3d3ac..3bf307d475 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -4379,7 +4379,7 @@ int32_t functionCompatList[] = { 6, 8, 7, }; -SAggFunctionInfo aggFunc[34] = {{ +SAggFunctionInfo aggFunc[35] = {{ // 0, count function does not invoke the finalize function "count", FUNCTION_TYPE_AGG, @@ -4820,4 +4820,18 @@ SAggFunctionInfo aggFunc[34] = {{ blockinfo_func_finalizer, block_func_merge, dataBlockRequired, - }}; + }, + { + // 34 + "cov", // return table id and the corresponding tags for join match and subscribe + FUNCTION_TYPE_AGG, + FUNCTION_COV, + FUNCTION_COV, + FUNCSTATE_SO | FUNCSTATE_STABLE, + function_setup, + sum_function, + function_finalizer, + sum_func_merge, + statisRequired, + } + }; diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 0f5acf5bcc..f567f6553b 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -39,6 +39,9 @@ extern "C" { TAOS_FIELD createField(const SSchema* pSchema); SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); +SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); + +SSourceParam addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); SInternalField* insertFieldInfo(SFieldInfo* pFieldInfo, int32_t index, SSchema* field); int32_t getNumOfFields(SFieldInfo* pFieldInfo); diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 6edcc4b225..448b742dd8 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -28,7 +28,8 @@ int32_t getNumOfTags(const STableMeta* pTableMeta); SSchema *getTableColumnSchema(const STableMeta *pTableMeta); SSchema *getTableTagSchema(const STableMeta* pTableMeta); -size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo); +SArray *getCurrentExprList(SQueryStmtInfo* pQueryInfo); +size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo); SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema); void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t level); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index d7e367cd0f..8ef2479a30 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1708,35 +1708,37 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f } } -SArray* getCurrentExprList(SQueryStmtInfo* pQueryInfo) { - assert(pQueryInfo != NULL && pQueryInfo->exprListLevelIndex >= 0 && pQueryInfo->exprListLevelIndex < 10); - return pQueryInfo->exprList[pQueryInfo->exprListLevelIndex]; -} -SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, - SSchema* pColSchema, SSchema* pResultSchema, tExprNode* pExprNode, int32_t interSize, const char* token, bool finalResult) { - STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); - - SExprInfo* pExpr = createExprInfo(pTableMetaInfo, functionId, pIndex, pExprNode, pResultSchema, interSize); +SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, int16_t functionId, SSourceParam* pSourceParam, int32_t outputIndex, + STableMetaInfo* pTableMetaInfo, SSchema* pResultSchema, int32_t interSize, const char* token, bool finalResult) { + SExprInfo* pExpr = createExprInfo(pTableMetaInfo, functionId, pSourceParam, pResultSchema, interSize); SArray* pExprList = getCurrentExprList(pQueryInfo); - addExprInfo(pExprList, outputColIndex, pExpr, pQueryInfo->exprListLevelIndex); + addExprInfo(pExprList, outputIndex, pExpr, pQueryInfo->exprListLevelIndex); tstrncpy(pExpr->base.token, token, sizeof(pExpr->base.token)); uint64_t uid = pTableMetaInfo->pTableMeta->uid; - if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) { - SArray* p = TSDB_COL_IS_TAG(pIndex->type) ? pTableMetaInfo->tagColList : pQueryInfo->colList; - columnListInsert(p, uid, pColSchema, pIndex->type); - } + if (pSourceParam->pColumnList != NULL) { + SColumn* pCol = taosArrayGetP(pSourceParam->pColumnList, 0); - pExpr->base.pColumns->flag = pIndex->type; - if (TSDB_COL_IS_NORMAL_COL(pIndex->type)) { - insertPrimaryTsColumn(pQueryInfo->colList, uid); + if (pCol->flag != COLUMN_INDEX_INITIAL_VAL) { + SArray* p = TSDB_COL_IS_TAG(pCol->flag) ? pTableMetaInfo->tagColList : pQueryInfo->colList; + + for (int32_t i = 0; i < pSourceParam->num; ++i) { + SColumn* pColumn = taosArrayGetP(pSourceParam->pColumnList, i); + SSchema s = createSchema(pColumn->info.type, pColumn->info.bytes, pColumn->info.colId, pColumn->name) ; + columnListInsert(p, uid, &s, pCol->flag); + } + } + + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { + insertPrimaryTsColumn(pQueryInfo->colList, uid); + } } if (finalResult) { - addResColumnInfo(pQueryInfo, outputColIndex, pResultSchema, pExpr); + addResColumnInfo(pQueryInfo, outputIndex, pResultSchema, pExpr); } return pExpr; @@ -1758,7 +1760,15 @@ static int32_t addOneExprInfo(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, i getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resInfo, 0, false); SSchema resultSchema = createSchema(resInfo.type, resInfo.bytes, getNewResColId(), name); - doAddOneExprInfo(pQueryInfo, outputIndex, functionId, pColIndex, pSchema, &resultSchema, pNode, resInfo.intermediateBytes, name, finalResult); + + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pColIndex->tableIndex); + + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pSchema->name, pColIndex->type, pSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, pNode, &c); + + doAddOneExprInfo(pQueryInfo, functionId, ¶m, outputIndex, pTableMetaInfo, &resultSchema, resInfo.intermediateBytes, name, finalResult); return TSDB_CODE_SUCCESS; } @@ -1799,7 +1809,12 @@ static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTab SColumnIndex indexTS = {.tableIndex = tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_ID, .type = TSDB_COL_NORMAL}; SSchema s = createSchema(TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(), "ts"); - SExprInfo* pExpr = createExprInfo(pTableMetaInfo, FUNCTION_TS_DUMMY, &indexTS, NULL, &s, TSDB_KEYSIZE); + SColumn col = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_NORMAL, &s); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &col); + + SExprInfo* pExpr = createExprInfo(pTableMetaInfo, FUNCTION_TS_DUMMY, ¶m, &s, TSDB_KEYSIZE); strncpy(pExpr->base.token, "ts", tListLen(pExpr->base.token)); SArray* pExprList = getCurrentExprList(pQueryInfo); @@ -1909,7 +1924,7 @@ static int32_t doHandleOneParam(SQueryStmtInfo *pQueryInfo, tSqlExprItem* pItem, } static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumnList, SMsgBuf* pMsgBuf); -static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf); +static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf); int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId, STableMetaInfo** pTableMetaInfo, SSchema* columnSchema, tExprNode** pNode, SColumnIndex* pIndex, @@ -1943,7 +1958,7 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId // todo extract the table uid pIndex->tableIndex = 0; - int32_t code = createComplexExpr(pQueryInfo, n, pParamElem, pMsgBuf); + int32_t code = addScalarExprAndResColumn(pQueryInfo, n, pParamElem, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1971,7 +1986,7 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId SArray* pExprList = getCurrentExprList(pQueryInfo); size_t n = taosArrayGetSize(pExprList); - int32_t code = createComplexExpr(pQueryInfo, n, pParamElem, pMsgBuf); + int32_t code = addScalarExprAndResColumn(pQueryInfo, n, pParamElem, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2048,8 +2063,13 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq char token[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, s.name, token,sizeof(s.name) - 1); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &columnSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, pNode, &c); + int32_t outputIndex = getNumOfFields(&pQueryInfo->fieldsInfo); - doAddOneExprInfo(pQueryInfo, outputIndex, functionId, &index, &columnSchema, &s, pNode, size, token, finalResult); + doAddOneExprInfo(pQueryInfo, functionId, ¶m, outputIndex, pTableMetaInfo, &s, size, token, finalResult); return TSDB_CODE_SUCCESS; } @@ -2116,7 +2136,15 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq char token[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, s.name, token, sizeof(s.name) - 1); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, numOfOutput, functionId, &index, &columnSchema, &s, pNode, resInfo.intermediateBytes, token, finalResult); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &columnSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, pNode, &c); + + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, functionId, ¶m, numOfOutput, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); + if (finalResult) { + addResColumnInfo(pQueryInfo, numOfOutput, &s, pExpr); + } if (functionId == FUNCTION_LEASTSQR) { // set the leastsquares parameters char val[8] = {0}; @@ -2251,7 +2279,13 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq char token[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, s.name, token, sizeof(s.name) - 1); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, colIndex, functionId, &index, &columnSchema, &s, pNode, resInfo.intermediateBytes, token, finalResult); + + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &columnSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, pNode, &c); + + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); SToken* pParamToken = &pParamElem[1].pNode->exprToken; pExpr->base.numOfParams += 1; @@ -2325,23 +2359,19 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq columnListInsert(pTableMetaInfo->tagColList, pTableMetaInfo->pTableMeta->uid, &pSchema[index.columnIndex], TSDB_COL_TAG); SSchema* pTagSchema = getTableTagSchema(pTableMetaInfo->pTableMeta); - SSchema s = {0}; - if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = *getTbnameColumnSchema(); - } else { - s = pTagSchema[index.columnIndex]; - } + SSchema s = (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX)? *getTbnameColumnSchema(): pTagSchema[index.columnIndex]; SResultDataInfo resInfo = {0}; int32_t ret = getResultDataInfo(s.type, s.bytes, FUNCTION_TID_TAG, 0, &resInfo, 0, 0); assert(ret == TSDB_CODE_SUCCESS); - s.type = (uint8_t)resInfo.type; - s.bytes = resInfo.bytes; - s.colId = getNewResColId(); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - - doAddOneExprInfo(pQueryInfo, 0, FUNCTION_TID_TAG, &index, &s, &s, NULL, 0, s.name, true); + SSchema result = createSchema(resInfo.type, resInfo.bytes, getNewResColId(), s.name); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &result); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &c); + + /*SExprInfo* pExpr = */doAddOneExprInfo(pQueryInfo, FUNCTION_TID_TAG, ¶m, 0, pTableMetaInfo, &result, 0, s.name, true); return TSDB_CODE_SUCCESS; } @@ -2362,14 +2392,81 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq char token[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, s.name, token, sizeof(s.name) - 1); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, colIndex, functionId, &index, &colSchema, &s, NULL, resInfo.intermediateBytes, token, finalResult); + + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &colSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &c); + + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); int64_t rowSize = pTableMetaInfo->pTableMeta->tableInfo.rowSize; addExprInfoParam(&pExpr->base, (char*) &rowSize, TSDB_DATA_TYPE_BIGINT, 8); return TSDB_CODE_SUCCESS; } + case FUNCTION_COV: { + // 1. valid the number of parameters + // no parameters or more than one parameter for function + if ((code = checkForkParam(pItem->pNode, 2, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } + tSqlExprItem* p1 = taosArrayGet(pItem->pNode->Expr.paramList, 0); + tSqlExprItem* p2 = taosArrayGet(pItem->pNode->Expr.paramList, 1); + + int32_t p1Type = p1->pNode->tokenId, p2Type = p2->pNode->tokenId; + if (p1Type != TK_ID || p2Type != TK_ID) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + // validate the first parameter + tExprNode* pNode1 = NULL; + int32_t tokenId1 = p1->pNode->tokenId; + SColumnIndex index1 = COLUMN_INDEX_INITIALIZER; + SSchema columnSchema1 = {0}; + + pQueryInfo->exprListLevelIndex += 1; + + code = extractFunctionParameterInfo(pQueryInfo, tokenId1, &pTableMetaInfo, &columnSchema1, &pNode1, &index1, p1, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // validate the second parameter + tExprNode* pNode2 = NULL; + int32_t tokenId2 = p1->pNode->tokenId; + SColumnIndex index2 = COLUMN_INDEX_INITIALIZER; + SSchema columnSchema2 = {0}; + code = extractFunctionParameterInfo(pQueryInfo, tokenId2, &pTableMetaInfo, &columnSchema2, &pNode2, &index2, p1, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pQueryInfo->exprListLevelIndex -= 1; + + int32_t srcType1 = columnSchema1.type, srcType2 = columnSchema2.type; + if (IS_VAR_DATA_TYPE(srcType1) || IS_VAR_DATA_TYPE(columnSchema2.type) || srcType1 == TSDB_DATA_TYPE_TIMESTAMP || + srcType1 == TSDB_DATA_TYPE_BOOL || srcType2 == TSDB_DATA_TYPE_TIMESTAMP || srcType2 == TSDB_DATA_TYPE_BOOL) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + SResultDataInfo resInfo = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double), .intermediateBytes = 0}; + SSchema s = createSchema(resInfo.type, resInfo.bytes, getNewResColId(), ""); + + char token[TSDB_COL_NAME_LEN] = {0}; + setTokenAndResColumnName(pItem, s.name, token, sizeof(s.name) - 1); + + SColumn c1 = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index1.type, &columnSchema1); + SColumn c2 = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index2.type, &columnSchema2); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, pNode1, &c1); + addIntoSourceParam(¶m, pNode2, &c2); + + doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); + return TSDB_CODE_SUCCESS; + } default: { // pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n); @@ -2406,7 +2503,13 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq char token[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, s.name, token, sizeof(s.name) - 1); - doAddOneExprInfo(pQueryInfo, colIndex, functionId, &index, colSchema, &s, NULL, resInfo.intermediateBytes, token, finalResult); + + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, colSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &c); + + doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); return TSDB_CODE_SUCCESS; } } @@ -2429,8 +2532,7 @@ static int32_t validateExprLeafColumnNode(SQueryStmtInfo *pQueryInfo, SToken* pC return TSDB_CODE_TSC_INVALID_OPERATION; } - SColumn c = {0}; - setColumn(&c, pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); + SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); taosArrayPush(pList, &c); return TSDB_CODE_SUCCESS; @@ -2451,7 +2553,7 @@ static int32_t validateExprLeafFunctionNode(SQueryStmtInfo* pQueryInfo, tSqlExpr if (scalar) { printf("scalar function found!\n"); -// if (createComplexExpr(pQueryInfo, outputIndex, &item, pMsgBuf) != TSDB_CODE_SUCCESS) { +// if (addScalarExprAndResColumn(pQueryInfo, outputIndex, &item, pMsgBuf) != TSDB_CODE_SUCCESS) { // return TSDB_CODE_TSC_INVALID_OPERATION; // } } else { @@ -2544,7 +2646,14 @@ SExprInfo* doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S const char* name = (aliasName == NULL)? pSchema->name:aliasName; SSchema s = createSchema(pSchema->type, pSchema->bytes, colId, name); - return doAddOneExprInfo(pQueryInfo, outputColIndex, functionId, &index, pSchema, &s, NULL, 0, pSchema->name, true); + + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &c); + + return doAddOneExprInfo(pQueryInfo, functionId, ¶m, outputColIndex, pTableMetaInfo, &s, 0, s.name, true); } static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -2636,7 +2745,13 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* char rawName[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, resultSchema.name, rawName, sizeof(colSchema.name) - 1); - doAddOneExprInfo(pQueryInfo, startPos, functionId, pIndex, &colSchema, &resultSchema, NULL, 0, rawName, true); + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, pIndex->type, &colSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &c); + + doAddOneExprInfo(pQueryInfo, functionId, ¶m, startPos, pTableMetaInfo, &colSchema, 0, rawName, true); return TSDB_CODE_SUCCESS; } @@ -2671,7 +2786,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* } // add the primary timestamp column even though it is not required by user - STableMeta* pTableMeta = pQueryInfo->pTableMetaInfo[index.tableIndex]->pTableMeta; + STableMeta* pTableMeta = getMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; if (pTableMeta->tableType != TSDB_TEMP_TABLE) { insertPrimaryTsColumn(pQueryInfo->colList, pTableMeta->uid); } @@ -2681,8 +2796,14 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* char token[TSDB_COL_NAME_LEN] = {0}; tstrncpy(token, pItem->pNode->exprToken.z, MIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN)); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, startPos, FUNCTION_PRJ, &index, &colSchema, &colSchema, NULL, 0, token, true); + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &colSchema); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, &c); + + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, FUNCTION_PRJ, ¶m, startPos, pTableMetaInfo, &colSchema, 0, token, true); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; taosVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); @@ -2801,25 +2922,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt tExprNode* pLeft = NULL; tExprNode* pRight= NULL; -// if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { -// // assert it is a scalar function -// *pExpr = calloc(1, sizeof(tExprNode)); -// (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; -// (*pExpr)->_function.num = 1; -// (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); -// -// SArray* pParamList = pSqlExpr->Expr.paramList; -// size_t num = taosArrayGetSize(pParamList); -// (*pExpr)->_function.pChild = calloc(num, POINTER_BYTES); -// -// for(int32_t i = 0; i < num; ++i) { -// tSqlExprItem* pItem = taosArrayGet(pParamList, i); -// sqlExprToExprNode(&(((*pExpr)->_function.pChild)[0]), pItem->pNode, pQueryInfo, pCols, pMsgBuf); -// } -// -// return TSDB_CODE_SUCCESS; -// } - SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pSqlExpr->type == SQL_NODE_EXPR) { if (pSqlExpr->pLeft != NULL) { @@ -2877,7 +2979,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt (*pExpr)->_function.pChild = p; (*pExpr)->_function.functionId = functionId; - (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n + 1); + (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); return TSDB_CODE_SUCCESS; } else { printf("agg function found, %s\n", pSqlExpr->exprToken.z); @@ -2941,8 +3043,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt assert(found); if (pCols != NULL) { // record the involved columns - SColumn c = {0}; - setColumn(&c, uid, NULL, TSDB_COL_NORMAL, (*pExpr)->pSchema); + SColumn c = createColumn(uid, NULL, TSDB_COL_NORMAL, (*pExpr)->pSchema); taosArrayPush(pCols, &c); } @@ -3042,23 +3143,20 @@ static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumn return TSDB_CODE_SUCCESS; } -static int32_t createComplexExpr(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { +static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid column name, illegal column type, or columns in expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; int32_t arithmeticType = NON_ARITHMEIC_EXPR; SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); -// if (validateComplexExpr(pItem->pNode, pQueryInfo, pColumnList, &arithmeticType, pMsgBuf) != TSDB_CODE_SUCCESS) { -// return buildInvalidOperationMsg(pMsgBuf, msg1); -// } - if (arithmeticType == NORMAL_ARITHMETIC) { // expr string is set as the parameter of function SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColumn)); + int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, colList, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); @@ -3198,7 +3296,7 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, } if (scalarFunc) { // scalar function - if ((code = createComplexExpr(pQueryInfo, outputIndex, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { + if ((code = addScalarExprAndResColumn(pQueryInfo, outputIndex, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { return code; } } else { // aggregate function @@ -3214,7 +3312,7 @@ int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, return code; } } else if (type == SQL_NODE_EXPR) { - if ((code = createComplexExpr(pQueryInfo, i, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { + if ((code = addScalarExprAndResColumn(pQueryInfo, i, pItem, pMsgBuf)) != TSDB_CODE_SUCCESS) { return code; } } else { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index c8e668f209..a3a6e58785 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -527,6 +527,44 @@ void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t fla } } +SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema) { + SColumn c; + c.uid = uid; + c.flag = flag; + c.info.colId = pSchema->colId; + c.info.bytes = pSchema->bytes; + c.info.type = pSchema->type; + + if (tableName != NULL) { + snprintf(c.name, tListLen(c.name), "%s.%s", tableName, pSchema->name); + } else { + tstrncpy(c.name, pSchema->name, tListLen(c.name)); + } + + return c; +} + +SSourceParam addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn) { + assert(pSourceParam != NULL); + pSourceParam->num += 1; + + if (pSourceParam->pExprNodeList != NULL) { + assert(pNode != NULL && pColumn == NULL); + if (pSourceParam->pExprNodeList == NULL) { + pSourceParam->pExprNodeList = taosArrayInit(4, POINTER_BYTES); + } + + taosArrayPush(pSourceParam->pExprNodeList, &pNode); + } else { + assert(pColumn != NULL); + if (pSourceParam->pColumnList == NULL) { + pSourceParam->pColumnList = taosArrayInit(4, POINTER_BYTES); + } + + taosArrayPush(pSourceParam->pColumnList, &pColumn); + } +} + int32_t getNumOfFields(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index df58651179..8d5fc9e9dc 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -18,8 +18,14 @@ SSchema* getTbnameColumnSchema() { return &_s; } +SArray* getCurrentExprList(SQueryStmtInfo* pQueryInfo) { + assert(pQueryInfo != NULL && pQueryInfo->exprListLevelIndex >= 0 && pQueryInfo->exprListLevelIndex < 10); + return pQueryInfo->exprList[pQueryInfo->exprListLevelIndex]; +} + size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo) { - return taosArrayGetSize(pQueryInfo->exprList[0]); + SArray* pExprList = getCurrentExprList(pQueryInfo); + return taosArrayGetSize(pExprList); } SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { @@ -55,21 +61,29 @@ SSchema* getTableTagSchema(const STableMeta* pTableMeta) { return getOneColumnSchema(pTableMeta, getTableInfo(pTableMeta).numOfColumns); } -static tExprNode* createFunctionExprNode(int32_t functionId, SSchema* pSchema, tExprNode* pColumnNode, int32_t numOfCols) { - if (pColumnNode == NULL) { - pColumnNode = calloc(1, sizeof(tExprNode)); - pColumnNode->nodeType = TEXPR_COL_NODE; - pColumnNode->pSchema = calloc(1, sizeof(SSchema)); - memcpy(pColumnNode->pSchema, pSchema, sizeof(SSchema)); +static tExprNode* createFunctionExprNode(int32_t functionId, struct SSourceParam *pParam) {//SSchema* pSchema, tExprNode* pColumnNode, int32_t numOfCols) { + tExprNode** p = malloc(pParam->num * POINTER_BYTES); + + if (pParam->pColumnList != NULL) { + for(int32_t i = 0; i < pParam->num; ++i) { + p[i] = calloc(1, sizeof(tExprNode)); + p[i]->nodeType = TEXPR_COL_NODE; + p[i]->pSchema = calloc(1, sizeof(SSchema)); + memcpy(p[i]->pSchema, taosArrayGetP(pParam->pColumnList, i), sizeof(SSchema)); + } } else { - assert(pSchema == NULL); + assert(pParam->pColumnList == NULL); + for(int32_t i = 0; i < pParam->num; ++i) { + p[i] = taosArrayGetP(pParam->pExprNodeList, i); + } } tExprNode* pNode = calloc(1, sizeof(tExprNode)); + pNode->nodeType = TEXPR_FUNCTION_NODE; pNode->_function.functionId = functionId; - pNode->_function.pChild = pColumnNode; - pNode->_function.num = numOfCols; + pNode->_function.pChild = p; + pNode->_function.num = pParam->num; return pNode; } @@ -87,7 +101,7 @@ SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema) { return pExpr; } -SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize) { +SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SSourceParam* pSourceParam, SSchema* pResSchema, int16_t interSize) { SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); if (pExpr == NULL) { return NULL; @@ -99,43 +113,36 @@ SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SC } SSqlExpr* p = &pExpr->base; - p->pColumns = calloc(1, sizeof(SColumn)); - p->numOfCols = 1; - if (pParamExpr != NULL) { - pExpr->pExpr = createFunctionExprNode(functionId, NULL, pParamExpr, 1); - // todo set the correct number of columns - } else if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - assert(pParamExpr == NULL); + p->pColumns = calloc(pSourceParam->num, sizeof(SColumn)); + p->numOfCols = pSourceParam->num; + p->interBytes = interSize; + memcpy(&p->resSchema, pResSchema, sizeof(SSchema)); + + if (pSourceParam->pExprNodeList != NULL) { + pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); + return pExpr; + } + + SColumn* pCol = taosArrayGetP(pSourceParam->pColumnList, 0); + if (pCol->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + assert(pSourceParam->num == 1); SSchema* s = getTbnameColumnSchema(); setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_TAG, s); - pExpr->pExpr = createFunctionExprNode(functionId, s, pParamExpr, 1); - } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX || functionId == FUNCTION_BLKINFO) { - assert(pParamExpr == NULL); + pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); + } else if (TSDB_COL_IS_UD_COL(pCol->flag) || functionId == FUNCTION_BLKINFO) { setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_UDC, pResSchema); - - SSchema s = createSchema(pResSchema->type, pResSchema->bytes, pColIndex->columnIndex, pResSchema->name); - pExpr->pExpr = createFunctionExprNode(functionId, &s, pParamExpr, 1); + pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); } else { - if (TSDB_COL_IS_TAG(pColIndex->type)) { - SSchema* pSchema = getTableTagSchema(pTableMetaInfo->pTableMeta); - setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_TAG, &pSchema[pColIndex->columnIndex]); - pExpr->pExpr = createFunctionExprNode(functionId, &pSchema[pColIndex->columnIndex], pParamExpr, 1); - } else if (pTableMetaInfo->pTableMeta != NULL) { - // in handling select database/version/server_status(), the pTableMeta is NULL - SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex); - setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_NORMAL, pSchema); - - pExpr->pExpr = createFunctionExprNode(functionId, pSchema, pParamExpr, 1); + for(int32_t i = 0; i < pSourceParam->num; ++i) { + SColumn* c = taosArrayGetP(pSourceParam->pColumnList, i); + p->pColumns[i] = *c; } + pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); } - p->pColumns->flag = pColIndex->type; - p->interBytes = interSize; - memcpy(&p->resSchema, pResSchema, sizeof(SSchema)); - return pExpr; } @@ -149,7 +156,7 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } - printf("add function, id:%d, level:%d\n", pExprInfo->pExpr->_function.functionId, level); + printf("add function, id:%d, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionId, level, taosArrayGetSize(pExprList)); } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 24b62d1313..a3916bbc06 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -394,14 +394,14 @@ void sqlCheck(const char* sql, bool valid) { //} TEST(testCase, function_Test10) { -// sqlCheck("select c from `t.1abc`", true); -// sqlCheck("select length(c) from `t.1abc`", true); -// sqlCheck("select sum(length(a+b)) from `t.1abc`", true); -// sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); -// sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); -// sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); -// sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); - sqlCheck("select concat(a,b) from `t.1abc`", true); + sqlCheck("select c from `t.1abc`", true); + sqlCheck("select length(c) from `t.1abc`", true); + sqlCheck("select sum(length(a+b)) from `t.1abc`", true); + sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); + sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); + sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); + sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); + sqlCheck("select cov(a, b) from `t.1abc`", true); // sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); // sqlCheck("select length(length(length(a))) from `t.1abc`", true); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 53631fc9f1..81404d74ca 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -191,7 +191,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, i); SSchema resultSchema = *pSchema; - SExprInfo* p = createExprInfo(pTableMetaInfo1, FUNCTION_PRJ, &index, NULL, &resultSchema, 0); + SExprInfo* p = NULL;//createExprInfo(pTableMetaInfo1, FUNCTION_PRJ, &index, NULL, &resultSchema, 0); pExpr[i] = p; } From b4bba7ee0db0e9e0a62b2ad8a6f4b8f0ac8368fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 10:32:06 +0800 Subject: [PATCH 09/26] [td-10564] Refactor the sql function parse procedure. --- include/libs/function/function.h | 4 ++-- include/libs/parser/parser.h | 2 +- source/libs/executor/src/executorimpl.c | 2 +- source/libs/function/src/taggfunction.c | 2 +- source/libs/function/src/tfill.c | 2 +- source/libs/parser/inc/queryInfoUtil.h | 2 +- source/libs/parser/src/astValidate.c | 10 ++++----- source/libs/parser/src/queryInfoUtil.c | 27 ++++++++++++------------- 8 files changed, 25 insertions(+), 26 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 8599b2a6f8..f4c1fbbbc0 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -182,8 +182,8 @@ typedef struct tExprNode { struct SVariant *pVal; // value node struct {// function node - char *functionName; - int32_t functionId; + char functionName[FUNCTIONS_NAME_MAX_LENGTH]; +// int32_t functionId; int32_t num; // Note that the attribute of pChild is not the parameter of function, it is the columns that involved in the diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 5457f0b957..29107b1b49 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -175,7 +175,7 @@ typedef struct SSourceParam { int32_t num; } SSourceParam; -SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); +SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7e19093b10..b54d53bcea 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -162,7 +162,7 @@ int64_t genQueryId(void) { static int32_t getExprFunctionId(SExprInfo *pExprInfo) { assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); - return pExprInfo->pExpr->_function.functionId; + return 0; } static void getNextTimeWindow(SQueryAttr* pQueryAttr, STimeWindow* tw) { diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 3bf307d475..dc6eadf7d8 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -4810,7 +4810,7 @@ SAggFunctionInfo aggFunc[35] = {{ }, { // 33 - "_block_dist", // return table id and the corresponding tags for join match and subscribe + "block_dist", // return table id and the corresponding tags for join match and subscribe FUNCTION_TYPE_AGG, FUNCTION_BLKINFO, FUNCTION_BLKINFO, diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index aa10523dea..9575416a03 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -544,7 +544,7 @@ struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, co pFillCol[i].col.colId = pExprInfo->base.resSchema.colId; pFillCol[i].tagIndex = -2; pFillCol[i].flag = pExprInfo->base.pColumns->flag; // always be the normal column for table query - pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; +// pFillCol[i].functionId = pExprInfo->pExpr->_function.functionId; pFillCol[i].fillVal.i = fillVal[i]; offset += pExprInfo->base.resSchema.bytes; diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 448b742dd8..5d51293939 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -43,7 +43,7 @@ void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32 void cleanupFieldInfo(SFieldInfo* pFieldInfo); STableComInfo getTableInfo(const STableMeta* pTableMeta); -SArray* extractFunctionIdList(SArray* pExprInfoList); +SArray *extractFunctionList(SArray* pExprInfoList); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 8ef2479a30..d38c0ad9b1 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1709,9 +1709,9 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f } -SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, int16_t functionId, SSourceParam* pSourceParam, int32_t outputIndex, +SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, const char* funcName, SSourceParam* pSourceParam, int32_t outputIndex, STableMetaInfo* pTableMetaInfo, SSchema* pResultSchema, int32_t interSize, const char* token, bool finalResult) { - SExprInfo* pExpr = createExprInfo(pTableMetaInfo, functionId, pSourceParam, pResultSchema, interSize); + SExprInfo* pExpr = createExprInfo(pTableMetaInfo, funcName, pSourceParam, pResultSchema, interSize); SArray* pExprList = getCurrentExprList(pQueryInfo); addExprInfo(pExprList, outputIndex, pExpr, pQueryInfo->exprListLevelIndex); @@ -2978,8 +2978,8 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; (*pExpr)->_function.pChild = p; - (*pExpr)->_function.functionId = functionId; - (*pExpr)->_function.functionName = strndup(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); +// (*pExpr)->_function.functionId = functionId; + strncpy((*pExpr)->_function.functionName, pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); return TSDB_CODE_SUCCESS; } else { printf("agg function found, %s\n", pSqlExpr->exprToken.z); @@ -3826,7 +3826,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer validateSqlNode(p, pQueryInfo, &buf); } - SArray* functionList = extractFunctionIdList(pQueryInfo->exprList[0]); + SArray* functionList = extractFunctionList(pQueryInfo->exprList[0]); extractFunctionDesc(functionList, &pQueryInfo->info); if ((code = checkForInvalidExpr(pQueryInfo, &buf)) != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 8d5fc9e9dc..c3364e812d 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -61,7 +61,7 @@ SSchema* getTableTagSchema(const STableMeta* pTableMeta) { return getOneColumnSchema(pTableMeta, getTableInfo(pTableMeta).numOfColumns); } -static tExprNode* createFunctionExprNode(int32_t functionId, struct SSourceParam *pParam) {//SSchema* pSchema, tExprNode* pColumnNode, int32_t numOfCols) { +static tExprNode* createFunctionExprNode(const char* funcName, struct SSourceParam *pParam) { tExprNode** p = malloc(pParam->num * POINTER_BYTES); if (pParam->pColumnList != NULL) { @@ -81,7 +81,7 @@ static tExprNode* createFunctionExprNode(int32_t functionId, struct SSourceParam tExprNode* pNode = calloc(1, sizeof(tExprNode)); pNode->nodeType = TEXPR_FUNCTION_NODE; - pNode->_function.functionId = functionId; + tstrncpy(pNode->_function.functionName, funcName, tListLen(pNode->_function.functionName)); pNode->_function.pChild = p; pNode->_function.num = pParam->num; @@ -101,7 +101,7 @@ SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema) { return pExpr; } -SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SSourceParam* pSourceParam, SSchema* pResSchema, int16_t interSize) { +SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSourceParam, SSchema* pResSchema, int16_t interSize) { SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); if (pExpr == NULL) { return NULL; @@ -120,7 +120,7 @@ SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SS memcpy(&p->resSchema, pResSchema, sizeof(SSchema)); if (pSourceParam->pExprNodeList != NULL) { - pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); + pExpr->pExpr = createFunctionExprNode(funcName, pSourceParam); return pExpr; } @@ -131,16 +131,16 @@ SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SS SSchema* s = getTbnameColumnSchema(); setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_TAG, s); - pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); - } else if (TSDB_COL_IS_UD_COL(pCol->flag) || functionId == FUNCTION_BLKINFO) { + pExpr->pExpr = createFunctionExprNode(funcName, pSourceParam); + } else if (TSDB_COL_IS_UD_COL(pCol->flag) || strcmp(funcName, "block_dist") == 0) { setColumn(p->pColumns, uid, pTableMetaInfo->aliasName, TSDB_COL_UDC, pResSchema); - pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); + pExpr->pExpr = createFunctionExprNode(funcName, pSourceParam); } else { for(int32_t i = 0; i < pSourceParam->num; ++i) { SColumn* c = taosArrayGetP(pSourceParam->pColumnList, i); p->pColumns[i] = *c; } - pExpr->pExpr = createFunctionExprNode(functionId, pSourceParam); + pExpr->pExpr = createFunctionExprNode(funcName, pSourceParam); } return pExpr; @@ -156,14 +156,13 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } - printf("add function, id:%d, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionId, level, taosArrayGetSize(pExprList)); + printf("add function, id:%s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { assert(pExprInfo != NULL); SSqlExpr* pse = &pExprInfo->base; - pExprInfo->pExpr->_function.functionId = functionId; assert(0); pse->resSchema.type = resType; @@ -172,7 +171,7 @@ void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index) { assert(pQueryInfo != NULL && pQueryInfo->exprList && index >= 0); - return taosArrayGetP(pQueryInfo->exprList, index); + return taosArrayGetP(getCurrentExprList(pQueryInfo->exprList), index); } void destroyExprInfo(SExprInfo* pExprInfo) { @@ -214,7 +213,7 @@ void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t byt int32_t getExprFunctionId(SExprInfo *pExprInfo) { assert(pExprInfo != NULL && pExprInfo->pExpr != NULL && pExprInfo->pExpr->nodeType == TEXPR_UNARYEXPR_NODE); - return pExprInfo->pExpr->_function.functionId; + return 0; } void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { @@ -310,14 +309,14 @@ int32_t getResRowLength(SArray* pExprList) { return size; } -SArray* extractFunctionIdList(SArray* pExprInfoList) { +SArray* extractFunctionList(SArray* pExprInfoList) { assert(pExprInfoList != NULL); size_t len = taosArrayGetSize(pExprInfoList); SArray* p = taosArrayInit(len, sizeof(int32_t)); for(int32_t i = 0; i < len; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pExprInfoList, i); - taosArrayPush(p, &pExprInfo->pExpr->_function.functionId); + taosArrayPush(p, &pExprInfo->pExpr->_function.functionName); } return p; From 4eebc5c47e296e2cac4c6fcd97580a8e6bdd68fe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 13:48:15 +0800 Subject: [PATCH 10/26] [td-10564] disable the function id during sql parse. --- include/common/taosmsg.h | 1 + include/libs/parser/parser.h | 2 - source/libs/parser/inc/queryInfoUtil.h | 2 +- source/libs/parser/src/astValidate.c | 222 ++++++++++-------------- source/libs/parser/src/queryInfoUtil.c | 19 +- source/libs/parser/test/parserTests.cpp | 166 +++++++++--------- 6 files changed, 198 insertions(+), 214 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2ccab41f8f..d26584d392 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -203,6 +203,7 @@ enum _mgmt_table { #define TSDB_COL_NORMAL 0x0u // the normal column of the table #define TSDB_COL_TAG 0x1u // the tag column type #define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_TMP 0x3u // internal column generated by the previous operators #define TSDB_COL_NULL 0x4u // the column filter NULL or not #define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 29107b1b49..c57d28ebf4 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -168,9 +168,7 @@ void columnListDestroy(SArray* pColumnList); void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel); typedef struct SSourceParam { -// struct tExprNode **pExprNode; SArray *pExprNodeList; //Array -// SColumn *pCols; SArray *pColumnList; //Array int32_t num; } SSourceParam; diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 5d51293939..eca3e07c64 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -30,7 +30,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); SArray *getCurrentExprList(SQueryStmtInfo* pQueryInfo); size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo); -SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema); +SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema, const char* funcName); void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t level); void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index d38c0ad9b1..da37dc2a1d 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1708,7 +1708,6 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f } } - SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, const char* funcName, SSourceParam* pSourceParam, int32_t outputIndex, STableMetaInfo* pTableMetaInfo, SSchema* pResultSchema, int32_t interSize, const char* token, bool finalResult) { SExprInfo* pExpr = createExprInfo(pTableMetaInfo, funcName, pSourceParam, pResultSchema, interSize); @@ -1722,7 +1721,7 @@ SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, const char* funcName, SS if (pSourceParam->pColumnList != NULL) { SColumn* pCol = taosArrayGetP(pSourceParam->pColumnList, 0); - if (pCol->flag != COLUMN_INDEX_INITIAL_VAL) { + if (TSDB_COL_IS_TAG(pCol->flag) || TSDB_COL_IS_NORMAL_COL(pCol->flag)) { SArray* p = TSDB_COL_IS_TAG(pCol->flag) ? pTableMetaInfo->tagColList : pQueryInfo->colList; for (int32_t i = 0; i < pSourceParam->num; ++i) { @@ -1744,6 +1743,12 @@ SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, const char* funcName, SS return pExpr; } +static void extractFunctionName(char* name, const tSqlExprItem* pItem) { + assert(pItem != NULL); + SToken* funcToken = &pItem->pNode->Expr.operand; + memcpy(name, funcToken->z, funcToken->n); +} + static int32_t addOneExprInfo(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, int32_t functionId, int32_t outputIndex, SSchema* pSchema, SColumnIndex* pColIndex, tExprNode* pNode, bool finalResult, SMsgBuf* pMsgBuf) { const char* msg1 = "not support column types"; if (functionId == FUNCTION_SPREAD) { @@ -1762,13 +1767,15 @@ static int32_t addOneExprInfo(SQueryStmtInfo* pQueryInfo, tSqlExprItem* pItem, i SSchema resultSchema = createSchema(resInfo.type, resInfo.bytes, getNewResColId(), name); STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pColIndex->tableIndex); - - SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pSchema->name, pColIndex->type, pSchema); + SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, pColIndex->type, pSchema); SSourceParam param = {0}; addIntoSourceParam(¶m, pNode, &c); - doAddOneExprInfo(pQueryInfo, functionId, ¶m, outputIndex, pTableMetaInfo, &resultSchema, resInfo.intermediateBytes, name, finalResult); + char fname[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + extractFunctionName(fname, pItem); + doAddOneExprInfo(pQueryInfo, fname, ¶m, outputIndex, pTableMetaInfo, &resultSchema, resInfo.intermediateBytes, name, finalResult); + return TSDB_CODE_SUCCESS; } @@ -1814,7 +1821,7 @@ static void setTsOutputExprInfo(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTab SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &col); - SExprInfo* pExpr = createExprInfo(pTableMetaInfo, FUNCTION_TS_DUMMY, ¶m, &s, TSDB_KEYSIZE); + SExprInfo* pExpr = createExprInfo(pTableMetaInfo, "ts_dummy", ¶m, &s, TSDB_KEYSIZE); strncpy(pExpr->base.token, "ts", tListLen(pExpr->base.token)); SArray* pExprList = getCurrentExprList(pQueryInfo); @@ -1908,6 +1915,7 @@ static int32_t doHandleOneParam(SQueryStmtInfo *pQueryInfo, tSqlExprItem* pItem, STableMetaInfo* pTableMetaInfo = {0}; int32_t code = extractFunctionParameterInfo(pQueryInfo, tokenId, &pTableMetaInfo, &columnSchema, &pNode, &index, pParamElem, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg3); } @@ -1937,6 +1945,8 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId const char* msg6 = "aggregate function can not be nested in aggregate function"; const char* msg7 = "invalid function name"; + pQueryInfo->exprListLevelIndex += 1; + if (tokenId == TK_ALL || tokenId == TK_ID) { // simple parameter // simple parameter or nested function // It is a parameter of a aggregate function, so it can not be still a aggregate function. @@ -1983,6 +1993,7 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId } } else if (tokenId == TK_PLUS || tokenId == TK_MINUS || tokenId == TK_STAR || tokenId == TK_REM || tokenId == TK_DIVIDE || tokenId == TK_CONCAT) { pIndex->tableIndex = 0; // todo set the correct table index + pIndex->type = TSDB_COL_TMP; // It is a temporary column generated by arithmetic expression. SArray* pExprList = getCurrentExprList(pQueryInfo); size_t n = taosArrayGetSize(pExprList); @@ -1998,7 +2009,8 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId } else { assert(0); } - + + pQueryInfo->exprListLevelIndex -= 1; return TSDB_CODE_SUCCESS; } @@ -2063,13 +2075,17 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq char token[TSDB_COL_NAME_LEN] = {0}; setTokenAndResColumnName(pItem, s.name, token,sizeof(s.name) - 1); + pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &columnSchema); SSourceParam param = {0}; addIntoSourceParam(¶m, pNode, &c); int32_t outputIndex = getNumOfFields(&pQueryInfo->fieldsInfo); - doAddOneExprInfo(pQueryInfo, functionId, ¶m, outputIndex, pTableMetaInfo, &s, size, token, finalResult); + + char fname[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + extractFunctionName(fname, pItem); + doAddOneExprInfo(pQueryInfo, fname, ¶m, outputIndex, pTableMetaInfo, &s, size, token, finalResult); return TSDB_CODE_SUCCESS; } @@ -2100,10 +2116,7 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq int32_t tokenId = pParamElem->pNode->tokenId; SColumnIndex index = COLUMN_INDEX_INITIALIZER; SSchema columnSchema = {0}; - - pQueryInfo->exprListLevelIndex += 1; code = extractFunctionParameterInfo(pQueryInfo, tokenId, &pTableMetaInfo, &columnSchema, &pNode, &index, pParamElem, pMsgBuf); - pQueryInfo->exprListLevelIndex -= 1; if (code != TSDB_CODE_SUCCESS) { return code; @@ -2141,11 +2154,10 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq SSourceParam param = {0}; addIntoSourceParam(¶m, pNode, &c); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, functionId, ¶m, numOfOutput, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); - if (finalResult) { - addResColumnInfo(pQueryInfo, numOfOutput, &s, pExpr); - } + char funcName[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + extractFunctionName(funcName, pItem); + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, funcName, ¶m, numOfOutput, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); if (functionId == FUNCTION_LEASTSQR) { // set the leastsquares parameters char val[8] = {0}; if (taosVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { @@ -2285,7 +2297,9 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq SSourceParam param = {0}; addIntoSourceParam(¶m, pNode, &c); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); + char funcName[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + extractFunctionName(funcName, pItem); + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, funcName, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); SToken* pParamToken = &pParamElem[1].pNode->exprToken; pExpr->base.numOfParams += 1; @@ -2426,8 +2440,6 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq SColumnIndex index1 = COLUMN_INDEX_INITIALIZER; SSchema columnSchema1 = {0}; - pQueryInfo->exprListLevelIndex += 1; - code = extractFunctionParameterInfo(pQueryInfo, tokenId1, &pTableMetaInfo, &columnSchema1, &pNode1, &index1, p1, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2443,8 +2455,6 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq return code; } - pQueryInfo->exprListLevelIndex -= 1; - int32_t srcType1 = columnSchema1.type, srcType2 = columnSchema2.type; if (IS_VAR_DATA_TYPE(srcType1) || IS_VAR_DATA_TYPE(columnSchema2.type) || srcType1 == TSDB_DATA_TYPE_TIMESTAMP || srcType1 == TSDB_DATA_TYPE_BOOL || srcType2 == TSDB_DATA_TYPE_TIMESTAMP || srcType2 == TSDB_DATA_TYPE_BOOL) { @@ -2464,7 +2474,10 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq addIntoSourceParam(¶m, pNode1, &c1); addIntoSourceParam(¶m, pNode2, &c2); - doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); + char funcName[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + extractFunctionName(funcName, pItem); + + doAddOneExprInfo(pQueryInfo, funcName, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); return TSDB_CODE_SUCCESS; } @@ -2634,14 +2647,14 @@ SExprInfo* doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S SSchema* pSchema = getOneColumnSchema(pTableMeta, pColIndex->columnIndex); SColumnIndex index = *pColIndex; - int16_t functionId = 0; + char* funcName = NULL; if (TSDB_COL_IS_TAG(index.type)) { int32_t numOfCols = getNumOfColumns(pTableMeta); index.columnIndex = pColIndex->columnIndex - numOfCols; - functionId = FUNCTION_TAGPRJ; + funcName = "project_tag"; } else { index.columnIndex = pColIndex->columnIndex; - functionId = FUNCTION_PRJ; + funcName = "project_col"; } const char* name = (aliasName == NULL)? pSchema->name:aliasName; @@ -2653,7 +2666,7 @@ SExprInfo* doAddProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - return doAddOneExprInfo(pQueryInfo, functionId, ¶m, outputColIndex, pTableMetaInfo, &s, 0, s.name, true); + return doAddOneExprInfo(pQueryInfo, funcName, ¶m, outputColIndex, pTableMetaInfo, &s, 0, s.name, true); } static int32_t doAddProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -2803,7 +2816,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, FUNCTION_PRJ, ¶m, startPos, pTableMetaInfo, &colSchema, 0, token, true); + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, "project", ¶m, startPos, pTableMetaInfo, &colSchema, 0, token, true); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; taosVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); @@ -3043,7 +3056,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt assert(found); if (pCols != NULL) { // record the involved columns - SColumn c = createColumn(uid, NULL, TSDB_COL_NORMAL, (*pExpr)->pSchema); + SColumn c = createColumn(uid, NULL, TSDB_COL_TMP, (*pExpr)->pSchema); taosArrayPush(pCols, &c); } @@ -3054,7 +3067,8 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } pQueryInfo->curTableIdx = index.tableIndex; - STableMeta* pTableMeta = getMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TEXPR_COL_NODE; @@ -3062,6 +3076,12 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; + + if (pCols != NULL) { // record the involved columns + SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_NORMAL, (*pExpr)->pSchema); + taosArrayPush(pCols, &c); + } + return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { int32_t colType = -1; @@ -3144,110 +3164,58 @@ static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumn } static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { - const char* msg1 = "invalid column name, illegal column type, or columns in expression from two tables"; - const char* msg2 = "invalid arithmetic expression in select clause"; - - int32_t arithmeticType = NON_ARITHMEIC_EXPR; - SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); - if (arithmeticType == NORMAL_ARITHMETIC) { - // expr string is set as the parameter of function - SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); - tExprNode* pNode = NULL; - SArray* colList = taosArrayInit(10, sizeof(SColumn)); + SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); + assert(taosArrayGetSize(pColumnList) == 0); - int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, colList, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(colList); - tExprTreeDestroy(pNode, NULL); - return buildInvalidOperationMsg(pMsgBuf, msg2); - } - - SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); - SArray* pExprList = getCurrentExprList(pQueryInfo); - addExprInfo(pExprList, exprIndex, pExpr, pQueryInfo->exprListLevelIndex); - setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); - - // check for if there is a tag in the arithmetic express - int32_t code = multiColumnListInsert(pQueryInfo, pColumnList, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(colList); - tExprTreeDestroy(pNode, NULL); - return code; - } - - SBufferWriter bw = tbufInitWriter(NULL, false); - -// TRY(0) { - exprTreeToBinary(&bw, pNode); -// } CATCH(code) { -// tbufCloseWriter(&bw); -// UNUSED(code); -// TODO: other error handling -// } END_TRY - - int32_t len = tbufTell(&bw); - char* c = tbufGetData(&bw, false); - - // set the serialized binary string as the parameter of arithmetic expression - addExprInfoParam(&pExpr->base, c, TSDB_DATA_TYPE_BINARY, (int32_t)len); - - // Need to be added to the final result list. - if (pQueryInfo->exprListLevelIndex == 0) { - addResColumnInfo(pQueryInfo, exprIndex, &pExpr->base.resSchema, pExpr); - } - - tbufCloseWriter(&bw); - taosArrayDestroy(colList); - } else { - SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); - addResColumnInfo(pQueryInfo, exprIndex, &s, NULL); - - assert(taosArrayGetSize(pColumnList) == 0); - - tExprNode* pNode = NULL; - int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, pMsgBuf); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(pNode, NULL); - return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); - } - - SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); - SArray* pExprList = getCurrentExprList(pQueryInfo); - addExprInfo(pExprList, exprIndex, pExpr, pQueryInfo->exprListLevelIndex); - - setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); - - // extract columns according to the tExprNode tree - size_t num = taosArrayGetSize(pColumnList); - pExpr->base.pColumns = calloc(num, sizeof(SColumn)); - for(int32_t i = 0; i < num; ++i) { - pExpr->base.pColumns[i] = *(SColumn*) taosArrayGet(pColumnList, i); - } - - pExpr->base.numOfCols = num; - - pExpr->base.numOfParams = 1; - SBufferWriter bw = tbufInitWriter(NULL, false); -// TRY(0) { - exprTreeToBinary(&bw, pExpr->pExpr); -// } CATCH(code) { -// tbufCloseWriter(&bw); -// UNUSED(code); -// TODO: other error handling -// } END_TRY - - SSqlExpr* pSqlExpr = &pExpr->base; - pSqlExpr->param[0].nLen = (int16_t) tbufTell(&bw); - pSqlExpr->param[0].pz = tbufGetData(&bw, true); - pSqlExpr->param[0].nType = TSDB_DATA_TYPE_BINARY; - - tbufCloseWriter(&bw); - -// tbufCloseWriter(&bw); // TODO there is a memory leak + tExprNode* pNode = NULL; + int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(pNode, NULL); + return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); } + SExprInfo* pExpr = createBinaryExprInfo(pNode, &s, "project"); + SArray* pExprList = getCurrentExprList(pQueryInfo); + addExprInfo(pExprList, exprIndex, pExpr, pQueryInfo->exprListLevelIndex); + + setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); + + // extract columns according to the tExprNode tree + size_t num = taosArrayGetSize(pColumnList); + pExpr->base.pColumns = calloc(num, sizeof(SColumn)); + for (int32_t i = 0; i < num; ++i) { + SColumn* pCol = taosArrayGet(pColumnList, i); + pExpr->base.pColumns[i] = *pCol; + + if (pCol->flag == TSDB_COL_NORMAL) { + SSchema sch = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); + columnListInsert(pQueryInfo->colList, pCol->uid, &sch, pCol->flag); + } + } + + pExpr->base.numOfCols = num; + + pExpr->base.numOfParams = 1; + SBufferWriter bw = tbufInitWriter(NULL, false); + // TRY(0) { + exprTreeToBinary(&bw, pExpr->pExpr); + // } CATCH(code) { + // tbufCloseWriter(&bw); + // UNUSED(code); + // TODO: other error handling + // } END_TRY + + SSqlExpr* pSqlExpr = &pExpr->base; + pSqlExpr->param[0].nLen = (int16_t)tbufTell(&bw); + pSqlExpr->param[0].pz = tbufGetData(&bw, true); + pSqlExpr->param[0].nType = TSDB_DATA_TYPE_BINARY; + + tbufCloseWriter(&bw); + + // tbufCloseWriter(&bw); // TODO there is a memory leak + taosArrayDestroy(pColumnList); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index c3364e812d..66f422ab29 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -68,8 +68,15 @@ static tExprNode* createFunctionExprNode(const char* funcName, struct SSourcePar for(int32_t i = 0; i < pParam->num; ++i) { p[i] = calloc(1, sizeof(tExprNode)); p[i]->nodeType = TEXPR_COL_NODE; - p[i]->pSchema = calloc(1, sizeof(SSchema)); - memcpy(p[i]->pSchema, taosArrayGetP(pParam->pColumnList, i), sizeof(SSchema)); + + SColumn* pSrc = taosArrayGetP(pParam->pColumnList, i); + SSchema* pSchema = calloc(1, sizeof(SSchema)); + + tstrncpy(pSchema->name, pSrc->name, tListLen(pSchema->name)); + pSchema->type = pSrc->info.type; + pSchema->bytes = pSrc->info.bytes; + pSchema->colId = pSrc->info.colId; + p[i]->pSchema = pSchema; } } else { assert(pParam->pColumnList == NULL); @@ -88,7 +95,7 @@ static tExprNode* createFunctionExprNode(const char* funcName, struct SSourcePar return pNode; } -SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema) { +SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema, const char* funcName) { assert(pNode != NULL && pResSchema != NULL); SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); @@ -97,6 +104,10 @@ SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema) { } pExpr->pExpr = pNode; + + char* fName = pExpr->pExpr->_function.functionName; + tstrncpy(fName, funcName, tListLen(pExpr->pExpr->_function.functionName)); + memcpy(&pExpr->base.resSchema, pResSchema, sizeof(SSchema)); return pExpr; } @@ -156,7 +167,7 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } - printf("add function, id:%s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); + printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index a3916bbc06..040b76590d 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include #include #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -393,86 +394,91 @@ void sqlCheck(const char* sql, bool valid) { // destroySqlInfo(&info1); //} -TEST(testCase, function_Test10) { - sqlCheck("select c from `t.1abc`", true); - sqlCheck("select length(c) from `t.1abc`", true); - sqlCheck("select sum(length(a+b)) from `t.1abc`", true); - sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); - sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); - sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); - sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); - sqlCheck("select cov(a, b) from `t.1abc`", true); -// sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); -// sqlCheck("select length(length(length(a))) from `t.1abc`", true); +//TEST(testCase, function_Test10) { +// sqlCheck("select c from `t.1abc`", true); +// sqlCheck("select length(c) from `t.1abc`", true); +// sqlCheck("select sum(length(a+b)) from `t.1abc`", true); +// sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); +// sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); +// sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); +// sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); +// sqlCheck("select cov(a, b) from `t.1abc`", true); +//// sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); +//// sqlCheck("select length(length(length(a))) from `t.1abc`", true); +//} + +TEST(testCase, function_Test6) { + SSqlInfo info1 = doGenerateAST("select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 5); + + SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); + ASSERT_EQ(p1->base.interBytes, 16); + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "sum"); + ASSERT_EQ(p1->pExpr->_function.num, 1); + + tExprNode* pParam = p1->pExpr->_function.pChild[0]; + + ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); + ASSERT_STREQ(pParam->pSchema->name, "t.1abc.a+b"); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); + + SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); + ASSERT_EQ(p2->base.pColumns->uid, 110); + ASSERT_EQ(p2->base.numOfParams, 0); + ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); + + ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STREQ(p2->base.pColumns->name, "t.1abc.b*a"); + + ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); + ASSERT_EQ(p2->base.interBytes, 24); + ASSERT_EQ(p2->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STRCASEEQ(p2->pExpr->_function.functionName, "first"); + ASSERT_EQ(p2->pExpr->_function.num, 1); + ASSERT_EQ(p2->pExpr->_function.pChild[0]->nodeType, TEXPR_COL_NODE); + ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); + + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); } -//TEST(testCase, function_Test6) { -// SSqlInfo info1 = doGenerateAST("select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_EQ(ret, 0); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 5); -// -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); -// ASSERT_EQ(p1->base.interBytes, 16); -// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); -// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_SUM); -// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); -// -// tExprNode* pParam = p1->pExpr->_node.pLeft; -// -// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); -// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_ADD); -// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_COL_NODE); -// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_COL_NODE); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); -// -// SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); -// ASSERT_EQ(p2->base.pColumns->uid, 110); -// ASSERT_EQ(p2->base.numOfParams, 0); -// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); -// ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); -// ASSERT_EQ(p2->base.interBytes, 24); -// ASSERT_EQ(p2->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); -// ASSERT_EQ(p2->pExpr->_function.functionId, FUNCTION_FIRST); -// ASSERT_TRUE(p2->pExpr->_node.pRight == NULL); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// + //TEST(testCase, function_Test7) { // SSqlInfo info1 = doGenerateAST("select count(a+b),count(1) from `t.1abc` interval(10s, 1s)"); // ASSERT_EQ(info1.valid, true); @@ -510,7 +516,7 @@ TEST(testCase, function_Test10) { // ASSERT_STRCASEEQ(p1->base.token, "count(a+b)"); // ASSERT_EQ(p1->base.interBytes, 8); // ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); -// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_COUNT); +// ASSERT_STREQ(p1->pExpr->_function.functionName, "count"); // ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); // // tExprNode* pParam = p1->pExpr->_node.pLeft; @@ -566,7 +572,7 @@ TEST(testCase, function_Test10) { // ASSERT_EQ(p1->base.interBytes, 16); // // ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); -// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); +// ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "top"); // ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); // // tExprNode* pParam = p1->pExpr->_node.pLeft; From 20406dd43b505c2dd84fe696e77e896f747610bc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 14:13:45 +0800 Subject: [PATCH 11/26] [td-10564]Fix bugs in parser. --- source/libs/parser/src/astValidate.c | 27 +-- source/libs/parser/test/parserTests.cpp | 270 ++++++++++++------------ 2 files changed, 146 insertions(+), 151 deletions(-) diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index da37dc2a1d..c342da7fb8 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2412,7 +2412,7 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); + SExprInfo* pExpr = doAddOneExprInfo(pQueryInfo, "block_dist", ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); int64_t rowSize = pTableMetaInfo->pTableMeta->tableInfo.rowSize; addExprInfoParam(&pExpr->base, (char*) &rowSize, TSDB_DATA_TYPE_BIGINT, 8); @@ -2724,7 +2724,7 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* const char* msg3 = "tbname not allowed in outer query"; SSchema colSchema = {0}; - int32_t functionId = 0; + char* funcName = NULL; if (outerQuery) { // todo?? STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); @@ -2746,10 +2746,10 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* } colSchema = pSchema[pIndex->columnIndex]; - functionId = FUNCTION_PRJ; + funcName = "project_col"; } else { colSchema = *getTbnameColumnSchema(); - functionId = FUNCTION_TAGPRJ; + funcName = "project_tag"; } SSchema resultSchema = colSchema; @@ -2764,7 +2764,7 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - doAddOneExprInfo(pQueryInfo, functionId, ¶m, startPos, pTableMetaInfo, &colSchema, 0, rawName, true); + doAddOneExprInfo(pQueryInfo, "project_tab", ¶m, startPos, pTableMetaInfo, &colSchema, 0, rawName, true); return TSDB_CODE_SUCCESS; } @@ -2991,7 +2991,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; (*pExpr)->_function.pChild = p; -// (*pExpr)->_function.functionId = functionId; strncpy((*pExpr)->_function.functionName, pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); return TSDB_CODE_SUCCESS; } else { @@ -3077,11 +3076,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; - if (pCols != NULL) { // record the involved columns - SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_NORMAL, (*pExpr)->pSchema); - taosArrayPush(pCols, &c); - } - + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { int32_t colType = -1; @@ -3188,11 +3183,6 @@ static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exp for (int32_t i = 0; i < num; ++i) { SColumn* pCol = taosArrayGet(pColumnList, i); pExpr->base.pColumns[i] = *pCol; - - if (pCol->flag == TSDB_COL_NORMAL) { - SSchema sch = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - columnListInsert(pQueryInfo->colList, pCol->uid, &sch, pCol->flag); - } } pExpr->base.numOfCols = num; @@ -3214,6 +3204,11 @@ static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exp tbufCloseWriter(&bw); + if (pQueryInfo->exprListLevelIndex == 0) { + int32_t exists = getNumOfFields(&pQueryInfo->fieldsInfo); + addResColumnInfo(pQueryInfo, exists, &pExpr->base.resSchema, pExpr); + } + // tbufCloseWriter(&bw); // TODO there is a memory leak taosArrayDestroy(pColumnList); diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 040b76590d..af0003d273 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -37,7 +37,7 @@ void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_ strcpy(p->name, name); } -void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { +void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq* req) { pQueryInfo->numOfTables = 1; pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES); @@ -55,7 +55,7 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { pTableMeta->tableInfo.rowSize = 28; pTableMeta->uid = 110; - pTableMetaInfo->tagColList = (SArray*) taosArrayInit(4, POINTER_BYTES); + pTableMetaInfo->tagColList = (SArray*)taosArrayInit(4, POINTER_BYTES); SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; setSchema(&pSchema[0], TSDB_DATA_TYPE_TIMESTAMP, 8, "ts", 0); @@ -75,8 +75,8 @@ void sqlCheck(const char* sql, bool valid) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); SMetaReq req = {0}; @@ -101,7 +101,7 @@ void sqlCheck(const char* sql, bool valid) { destroySqlInfo(&info1); } -} +} // namespace //TEST(testCase, validateAST_test) { // SSqlInfo info1 = doGenerateAST("select a a1111, a+b + 22, tbname from `t.1abc` where tsexprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 3); // -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->info.colId, 110); +// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); // ASSERT_EQ(p1->base.numOfParams, 0); // ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_INT); // ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1111"); // ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); // ASSERT_EQ(p1->base.pColumns->info.colId, 1); // ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "a"); +// ASSERT_STRCASEEQ(p1->base.token, "a1111"); // // ASSERT_EQ(taosArrayGetSize(pExprList), 3); // -// SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); -// ASSERT_EQ(p2->base.pColumns->uid, 0); +// SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); +// ASSERT_EQ(p2->base.pColumns->uid, 110); // ASSERT_EQ(p2->base.numOfParams, 1); // it is the serialized binary string of expression. // ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); // ASSERT_STRCASEEQ(p2->base.resSchema.name, "a+b + 22"); // -//// ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a"); -//// ASSERT_EQ(p1->base.colInfo.colId, 1); -//// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); +// // ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a"); +// // ASSERT_EQ(p1->base.colInfo.colId, 1); +// // ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); // ASSERT_STRCASEEQ(p2->base.token, "a+b + 22"); // // ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); @@ -170,8 +170,8 @@ void sqlCheck(const char* sql, bool valid) { // buf.len = 128; // buf.buf = msg; // -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); // ASSERT_EQ(code, 0); // // SMetaReq req = {0}; @@ -188,7 +188,7 @@ void sqlCheck(const char* sql, bool valid) { // SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 1); // -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); // ASSERT_EQ(p1->base.pColumns->uid, 110); // ASSERT_EQ(p1->base.numOfParams, 0); // ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); @@ -216,8 +216,8 @@ void sqlCheck(const char* sql, bool valid) { // buf.len = 128; // buf.buf = msg; // -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); // ASSERT_EQ(code, 0); // // SMetaReq req = {0}; @@ -234,7 +234,7 @@ void sqlCheck(const char* sql, bool valid) { // SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 1); // -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); // ASSERT_EQ(p1->base.pColumns->uid, 110); // ASSERT_EQ(p1->base.numOfParams, 0); // ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); @@ -262,8 +262,8 @@ void sqlCheck(const char* sql, bool valid) { // buf.len = 128; // buf.buf = msg; // -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); // ASSERT_EQ(code, 0); // // SMetaReq req = {0}; @@ -278,9 +278,9 @@ void sqlCheck(const char* sql, bool valid) { // ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); // // SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 4); +// ASSERT_EQ(taosArrayGetSize(pExprList), 6); // -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); // ASSERT_EQ(p1->base.pColumns->uid, 110); // ASSERT_EQ(p1->base.numOfParams, 0); // ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_TIMESTAMP); @@ -291,7 +291,7 @@ void sqlCheck(const char* sql, bool valid) { // ASSERT_STRCASEEQ(p1->base.token, "first(ts)"); // ASSERT_EQ(p1->base.interBytes, 24); // -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 4); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 6); // // destroyQueryInfo(pQueryInfo); // qParserClearupMetaRequestInfo(&req); @@ -299,7 +299,7 @@ void sqlCheck(const char* sql, bool valid) { //} // //TEST(testCase, function_Test4) { -// SSqlInfo info1 = doGenerateAST("select _block_dist() as a1 from `t.1abc`"); +// SSqlInfo info1 = doGenerateAST("select block_dist() as a1 from `t.1abc`"); // ASSERT_EQ(info1.valid, true); // // char msg[128] = {0}; @@ -307,8 +307,8 @@ void sqlCheck(const char* sql, bool valid) { // buf.len = 128; // buf.buf = msg; // -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); // ASSERT_EQ(code, 0); // // SMetaReq req = {0}; @@ -325,15 +325,15 @@ void sqlCheck(const char* sql, bool valid) { // SArray* pExprList = pQueryInfo->exprList[0]; // ASSERT_EQ(taosArrayGetSize(pExprList), 1); // -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); +// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); // ASSERT_EQ(p1->base.pColumns->uid, 110); // ASSERT_EQ(p1->base.numOfParams, 1); // ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BINARY); // ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -//// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); -//// ASSERT_EQ(p1->base.colInfo.colId, 0); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "_block_dist()"); +// // ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); +// // ASSERT_EQ(p1->base.colInfo.colId, 0); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_UDC); +// ASSERT_STRCASEEQ(p1->base.token, "block_dist()"); // ASSERT_EQ(p1->base.interBytes, 0); // // ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 1); @@ -344,71 +344,10 @@ void sqlCheck(const char* sql, bool valid) { // destroySqlInfo(&info1); //} -//TEST(testCase, function_Test5) { -// //todo select concat(concat(a, b), concat(b, a)) from `t.1abc`; -// -// SSqlInfo info1 = doGenerateAST("select sum(a) + avg(b) as a1 from `t.1abc`"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_EQ(ret, 0); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 3); -// -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.numOfCols, 2); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// -// ASSERT_EQ(p1->base.numOfParams, 1); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -// -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "sum(a) + avg(b)"); -// ASSERT_EQ(p1->base.interBytes, 0); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} +TEST(testCase, function_Test5) { + // todo select concat(concat(a, b), concat(b, a)) from `t.1abc`; -//TEST(testCase, function_Test10) { -// sqlCheck("select c from `t.1abc`", true); -// sqlCheck("select length(c) from `t.1abc`", true); -// sqlCheck("select sum(length(a+b)) from `t.1abc`", true); -// sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); -// sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); -// sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); -// sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); -// sqlCheck("select cov(a, b) from `t.1abc`", true); -//// sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); -//// sqlCheck("select length(length(length(a))) from `t.1abc`", true); -//} - -TEST(testCase, function_Test6) { - SSqlInfo info1 = doGenerateAST("select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); + SSqlInfo info1 = doGenerateAST("select sum(a) + avg(b) as a1 from `t.1abc`"); ASSERT_EQ(info1.valid, true); char msg[128] = {0}; @@ -416,8 +355,8 @@ TEST(testCase, function_Test6) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); - int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); SMetaReq req = {0}; @@ -433,53 +372,114 @@ TEST(testCase, function_Test6) { ASSERT_EQ(ret, 0); SArray* pExprList = pQueryInfo->exprList[0]; - ASSERT_EQ(taosArrayGetSize(pExprList), 5); + ASSERT_EQ(taosArrayGetSize(pExprList), 3); - SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.numOfCols, 2); ASSERT_EQ(p1->base.pColumns->uid, 110); - ASSERT_EQ(p1->base.numOfParams, 0); + + ASSERT_EQ(p1->base.numOfParams, 1); ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); - ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); - ASSERT_EQ(p1->base.interBytes, 16); - ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); - ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "sum"); - ASSERT_EQ(p1->pExpr->_function.num, 1); - - tExprNode* pParam = p1->pExpr->_function.pChild[0]; - - ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); - ASSERT_STREQ(pParam->pSchema->name, "t.1abc.a+b"); + ASSERT_STREQ(p1->base.pColumns->name, "sum(a)"); + ASSERT_STRCASEEQ(p1->base.token, "sum(a) + avg(b)"); + ASSERT_EQ(p1->base.interBytes, 0); ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); - ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); - - SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1); - ASSERT_EQ(p2->base.pColumns->uid, 110); - ASSERT_EQ(p2->base.numOfParams, 0); - ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); - - ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_TMP); - ASSERT_STREQ(p2->base.pColumns->name, "t.1abc.b*a"); - - ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); - ASSERT_EQ(p2->base.interBytes, 24); - ASSERT_EQ(p2->pExpr->nodeType, TEXPR_FUNCTION_NODE); - ASSERT_STRCASEEQ(p2->pExpr->_function.functionName, "first"); - ASSERT_EQ(p2->pExpr->_function.num, 1); - ASSERT_EQ(p2->pExpr->_function.pChild[0]->nodeType, TEXPR_COL_NODE); - ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); - + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); destroySqlInfo(&info1); } +//TEST(testCase, function_Test10) { +// sqlCheck("select c from `t.1abc`", true); +// sqlCheck("select length(c) from `t.1abc`", true); +// sqlCheck("select sum(length(a+b)) from `t.1abc`", true); +// sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); +// sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); +// sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); +// sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); +// sqlCheck("select cov(a, b) from `t.1abc`", true); +// // sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); +// // sqlCheck("select length(length(length(a))) from `t.1abc`", true); +//} +// +//TEST(testCase, function_Test6) { +// SSqlInfo info1 = doGenerateAST( +// "select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); +// ASSERT_EQ(info1.valid, true); +// +// char msg[128] = {0}; +// SMsgBuf buf; +// buf.len = 128; +// buf.buf = msg; +// +// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); +// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); +// ASSERT_EQ(code, 0); +// +// SMetaReq req = {0}; +// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); +// ASSERT_EQ(ret, 0); +// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); +// +// SQueryStmtInfo* pQueryInfo = createQueryInfo(); +// setTableMetaInfo(pQueryInfo, &req); +// +// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); +// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); +// ASSERT_EQ(ret, 0); +// +// SArray* pExprList = pQueryInfo->exprList[0]; +// ASSERT_EQ(taosArrayGetSize(pExprList), 5); +// +// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); +// ASSERT_EQ(p1->base.pColumns->uid, 110); +// ASSERT_EQ(p1->base.numOfParams, 0); +// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); +// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); +// ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); +// ASSERT_EQ(p1->base.interBytes, 16); +// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); +// ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "sum"); +// ASSERT_EQ(p1->pExpr->_function.num, 1); +// +// tExprNode* pParam = p1->pExpr->_function.pChild[0]; +// +// ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); +// ASSERT_STREQ(pParam->pSchema->name, "t.1abc.a+b"); +// +// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); +// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); +// +// SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); +// ASSERT_EQ(p2->base.pColumns->uid, 110); +// ASSERT_EQ(p2->base.numOfParams, 0); +// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); +// ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); +// +// ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_TMP); +// ASSERT_STREQ(p2->base.pColumns->name, "t.1abc.b*a"); +// +// ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); +// ASSERT_EQ(p2->base.interBytes, 24); +// ASSERT_EQ(p2->pExpr->nodeType, TEXPR_FUNCTION_NODE); +// ASSERT_STRCASEEQ(p2->pExpr->_function.functionName, "first"); +// ASSERT_EQ(p2->pExpr->_function.num, 1); +// ASSERT_EQ(p2->pExpr->_function.pChild[0]->nodeType, TEXPR_COL_NODE); +// ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); +// +// destroyQueryInfo(pQueryInfo); +// qParserClearupMetaRequestInfo(&req); +// destroySqlInfo(&info1); +//} -//TEST(testCase, function_Test7) { +// TEST(testCase, function_Test7) { // SSqlInfo info1 = doGenerateAST("select count(a+b),count(1) from `t.1abc` interval(10s, 1s)"); // ASSERT_EQ(info1.valid, true); // @@ -534,7 +534,7 @@ TEST(testCase, function_Test6) { // destroySqlInfo(&info1); //} // -//TEST(testCase, function_Test8) { +// TEST(testCase, function_Test8) { // SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); // ASSERT_EQ(info1.valid, true); // @@ -590,7 +590,7 @@ TEST(testCase, function_Test6) { // destroySqlInfo(&info1); //} // -//TEST(testCase, invalid_sql_Test) { +// TEST(testCase, invalid_sql_Test) { // char msg[128] = {0}; // SMsgBuf buf; // buf.len = 128; From f4bafd6f4c0f668e1bb4fa118b59d8f79d76a27a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 17:14:01 +0800 Subject: [PATCH 12/26] [td-10564] fix bug in parser nested function --- include/common/taosmsg.h | 5 +- source/libs/parser/inc/queryInfoUtil.h | 2 +- source/libs/parser/src/astValidate.c | 125 +-- source/libs/parser/src/queryInfoUtil.c | 12 +- source/libs/parser/test/parserTests.cpp | 970 ++++++++++++------------ 5 files changed, 569 insertions(+), 545 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index d26584d392..bd1964187d 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -203,15 +203,14 @@ enum _mgmt_table { #define TSDB_COL_NORMAL 0x0u // the normal column of the table #define TSDB_COL_TAG 0x1u // the tag column type #define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column -#define TSDB_COL_TMP 0x3u // internal column generated by the previous operators -#define TSDB_COL_NULL 0x4u // the column filter NULL or not +#define TSDB_COL_TMP 0x4u // internal column generated by the previous operators +#define TSDB_COL_NULL 0x8u // the column filter NULL or not #define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) #define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) - extern char *taosMsg[]; #pragma pack(push, 1) diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index eca3e07c64..5d51293939 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -30,7 +30,7 @@ SSchema *getTableTagSchema(const STableMeta* pTableMeta); SArray *getCurrentExprList(SQueryStmtInfo* pQueryInfo); size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo); -SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema, const char* funcName); +SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema); void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t level); void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index c342da7fb8..4098fa5392 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -1711,11 +1711,11 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, const char* funcName, SSourceParam* pSourceParam, int32_t outputIndex, STableMetaInfo* pTableMetaInfo, SSchema* pResultSchema, int32_t interSize, const char* token, bool finalResult) { SExprInfo* pExpr = createExprInfo(pTableMetaInfo, funcName, pSourceParam, pResultSchema, interSize); + tstrncpy(pExpr->base.token, token, sizeof(pExpr->base.token)); SArray* pExprList = getCurrentExprList(pQueryInfo); addExprInfo(pExprList, outputIndex, pExpr, pQueryInfo->exprListLevelIndex); - tstrncpy(pExpr->base.token, token, sizeof(pExpr->base.token)); uint64_t uid = pTableMetaInfo->pTableMeta->uid; if (pSourceParam->pColumnList != NULL) { @@ -1789,7 +1789,7 @@ static int32_t checkForAliasName(SMsgBuf* pMsgBuf, char* aliasName) { } static int32_t validateComplexExpr(tSqlExpr* pExpr, SQueryStmtInfo* pQueryInfo, SArray* pColList, int32_t* type, SMsgBuf* pMsgBuf); -static int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, SMsgBuf* pMsgBuf); +static int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, bool* keepTableCols, SMsgBuf* pMsgBuf); static int64_t getTickPerSecond(SVariant* pVariant, int32_t precision, int64_t* tickPerSec, SMsgBuf *pMsgBuf) { const char* msg10 = "derivative duration should be greater than 1 Second"; @@ -2385,7 +2385,7 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - /*SExprInfo* pExpr = */doAddOneExprInfo(pQueryInfo, FUNCTION_TID_TAG, ¶m, 0, pTableMetaInfo, &result, 0, s.name, true); + /*SExprInfo* pExpr = */doAddOneExprInfo(pQueryInfo, "tbid", ¶m, 0, pTableMetaInfo, &result, 0, s.name, true); return TSDB_CODE_SUCCESS; } @@ -2522,7 +2522,9 @@ int32_t addAggExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t colIndex, tSq SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - doAddOneExprInfo(pQueryInfo, functionId, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); + char funcName[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + extractFunctionName(funcName, pItem); + doAddOneExprInfo(pQueryInfo, funcName, ¶m, colIndex, pTableMetaInfo, &s, resInfo.intermediateBytes, token, finalResult); return TSDB_CODE_SUCCESS; } } @@ -2764,7 +2766,7 @@ static int32_t handleTbnameProjection(SQueryStmtInfo* pQueryInfo, tSqlExprItem* SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, &c); - doAddOneExprInfo(pQueryInfo, "project_tab", ¶m, startPos, pTableMetaInfo, &colSchema, 0, rawName, true); + doAddOneExprInfo(pQueryInfo, "project_tag", ¶m, startPos, pTableMetaInfo, &colSchema, 0, rawName, true); return TSDB_CODE_SUCCESS; } @@ -2931,21 +2933,49 @@ int32_t validateComplexExpr(tSqlExpr * pExpr, SQueryStmtInfo* pQueryInfo, SArray return TSDB_CODE_SUCCESS; } -int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, SMsgBuf* pMsgBuf) { +static uint64_t findTmpSourceColumnInNextLevel(SQueryStmtInfo* pQueryInfo, tExprNode *pExpr) { + // This function must be a aggregate function, so it must be in the next level + pQueryInfo->exprListLevelIndex += 1; + + // set the input column data byte and type. + SArray* pExprList = getCurrentExprList(pQueryInfo); + + bool found = false; + uint64_t uid = 0; + + size_t size = taosArrayGetSize(pExprList); + for (int32_t i = 0; i < size; ++i) { + SExprInfo* p1 = taosArrayGetP(pExprList, i); + + if (strcmp((pExpr)->pSchema->name, p1->base.resSchema.name) == 0) { + memcpy((pExpr)->pSchema, &p1->base.resSchema, sizeof(SSchema)); + found = true; + uid = p1->base.pColumns->uid; + break; + } + } + + assert(found); + pQueryInfo->exprListLevelIndex -= 1; + + return uid; +} + +int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, bool* keepTableCols, SMsgBuf* pMsgBuf) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pSqlExpr->type == SQL_NODE_EXPR) { if (pSqlExpr->pLeft != NULL) { - int32_t ret = sqlExprToExprNode(&pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, pMsgBuf); + int32_t ret = sqlExprToExprNode(&pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, keepTableCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } } if (pSqlExpr->pRight != NULL) { - int32_t ret = sqlExprToExprNode(&pRight, pSqlExpr->pRight, pQueryInfo, pCols, pMsgBuf); + int32_t ret = sqlExprToExprNode(&pRight, pSqlExpr->pRight, pQueryInfo, pCols, keepTableCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(pLeft, NULL); return ret; @@ -2960,27 +2990,32 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt SArray* pParamList = pSqlExpr->Expr.paramList; if (pParamList != NULL && taosArrayGetSize(pParamList) > 0) { - size_t num = taosArrayGetSize(pParamList); - - tExprNode** p = calloc(num, POINTER_BYTES); - pQueryInfo->exprListLevelIndex += 1; - - for(int32_t i = 0; i < num; ++i) { - tSqlExprItem* pItem = taosArrayGet(pParamList, i); - int32_t code = sqlExprToExprNode(&p[i], pItem->pNode, pQueryInfo, pCols, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - pQueryInfo->exprListLevelIndex -= 1; - bool scalar = false; int32_t functionId = qIsBuiltinFunction(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n, &scalar); if (functionId < 0) { return TSDB_CODE_TSC_INVALID_OPERATION; } + if (!scalar) { + pQueryInfo->exprListLevelIndex += 1; + } + + *keepTableCols = false; + + size_t num = taosArrayGetSize(pParamList); + tExprNode** p = calloc(num, POINTER_BYTES); + + pQueryInfo->exprListLevelIndex += 1; + + for(int32_t i = 0; i < num; ++i) { + tSqlExprItem* pItem = taosArrayGet(pParamList, i); + int32_t code = sqlExprToExprNode(&p[i], pItem->pNode, pQueryInfo, pCols, keepTableCols, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pQueryInfo->exprListLevelIndex -= 1; int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); if (scalar) { @@ -3000,6 +3035,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt return TSDB_CODE_TSC_INVALID_OPERATION; } + pQueryInfo->exprListLevelIndex -= 1; // convert the aggregate function to be the input data columns for the outer function. } } @@ -3035,30 +3071,11 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); strncpy((*pExpr)->pSchema->name, pSqlExpr->exprToken.z, pSqlExpr->exprToken.n); - // set the input column data byte and type. - SArray* pExprList = getCurrentExprList(pQueryInfo); - size_t size = taosArrayGetSize(pExprList); - - bool found = false; - uint64_t uid = 0; - for (int32_t i = 0; i < size; ++i) { - SExprInfo* p1 = taosArrayGetP(pExprList, i); - - if (strcmp((*pExpr)->pSchema->name, p1->base.resSchema.name) == 0) { - memcpy((*pExpr)->pSchema, &p1->base.resSchema, sizeof(SSchema)); - found = true; - uid = p1->base.pColumns->uid; - break; - } - } - - assert(found); - - if (pCols != NULL) { // record the involved columns - SColumn c = createColumn(uid, NULL, TSDB_COL_TMP, (*pExpr)->pSchema); + uint64_t uid = findTmpSourceColumnInNextLevel(pQueryInfo, *pExpr); + if (!(*keepTableCols)) { + SColumn c = createColumn(uid, NULL, TSDB_COL_TMP, (*pExpr)->pSchema); taosArrayPush(pCols, &c); } - } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column expression int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { @@ -3071,11 +3088,16 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TEXPR_COL_NODE; - (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); + (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; + if (*keepTableCols) { + SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, index.type, (*pExpr)->pSchema); + taosArrayPush(pCols, &c); + } + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { @@ -3160,23 +3182,22 @@ static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumn static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exprIndex, tSqlExprItem* pItem, SMsgBuf* pMsgBuf) { SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); - SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); - assert(taosArrayGetSize(pColumnList) == 0); tExprNode* pNode = NULL; - int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, pMsgBuf); + bool keepTableCols = true; + int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(pNode, NULL); return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); } - SExprInfo* pExpr = createBinaryExprInfo(pNode, &s, "project"); + SExprInfo* pExpr = createBinaryExprInfo(pNode, &s); + setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); + SArray* pExprList = getCurrentExprList(pQueryInfo); addExprInfo(pExprList, exprIndex, pExpr, pQueryInfo->exprListLevelIndex); - setTokenAndResColumnName(pItem, pExpr->base.resSchema.name, pExpr->base.token, TSDB_COL_NAME_LEN); - // extract columns according to the tExprNode tree size_t num = taosArrayGetSize(pColumnList); pExpr->base.pColumns = calloc(num, sizeof(SColumn)); diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 66f422ab29..91017ec631 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -95,7 +95,7 @@ static tExprNode* createFunctionExprNode(const char* funcName, struct SSourcePar return pNode; } -SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema, const char* funcName) { +SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema) { assert(pNode != NULL && pResSchema != NULL); SExprInfo* pExpr = calloc(1, sizeof(SExprInfo)); @@ -104,10 +104,6 @@ SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema, const cha } pExpr->pExpr = pNode; - - char* fName = pExpr->pExpr->_function.functionName; - tstrncpy(fName, funcName, tListLen(pExpr->pExpr->_function.functionName)); - memcpy(&pExpr->base.resSchema, pResSchema, sizeof(SSchema)); return pExpr; } @@ -167,7 +163,11 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } - printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); + if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { + printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); + } else { + printf("add operator: %s, level:%d, total:%ld\n", pExprInfo->base.resSchema.name, level, taosArrayGetSize(pExprList)); + } } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index af0003d273..79bdb561b7 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -103,246 +103,246 @@ void sqlCheck(const char* sql, bool valid) { } // namespace -//TEST(testCase, validateAST_test) { -// SSqlInfo info1 = doGenerateAST("select a a1111, a+b + 22, tbname from `t.1abc` where tsexprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 3); -// -// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_INT); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1111"); -// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); -// ASSERT_EQ(p1->base.pColumns->info.colId, 1); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "a1111"); -// -// ASSERT_EQ(taosArrayGetSize(pExprList), 3); -// -// SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); -// ASSERT_EQ(p2->base.pColumns->uid, 110); -// ASSERT_EQ(p2->base.numOfParams, 1); // it is the serialized binary string of expression. -// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p2->base.resSchema.name, "a+b + 22"); -// -// // ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a"); -// // ASSERT_EQ(p1->base.colInfo.colId, 1); -// // ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p2->base.token, "a+b + 22"); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// -//TEST(testCase, function_Test) { -// SSqlInfo info1 = doGenerateAST("select count(a) from `t.1abc`"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 1); -// -// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a)"); -// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); -// ASSERT_EQ(p1->base.pColumns->info.colId, 1); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "count(a)"); -// ASSERT_EQ(p1->base.interBytes, 8); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// -//TEST(testCase, function_Test2) { -// SSqlInfo info1 = doGenerateAST("select count(a) abc from `t.1abc`"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 1); -// -// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "abc"); -// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); -// ASSERT_EQ(p1->base.pColumns->info.colId, 1); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "count(a)"); -// ASSERT_EQ(p1->base.interBytes, 8); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// -//TEST(testCase, function_Test3) { -// SSqlInfo info1 = doGenerateAST("select first(*) from `t.1abc`"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 6); -// -// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_TIMESTAMP); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "first(ts)"); -// ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.ts"); -// ASSERT_EQ(p1->base.pColumns->info.colId, 0); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "first(ts)"); -// ASSERT_EQ(p1->base.interBytes, 24); -// -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 6); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// -//TEST(testCase, function_Test4) { -// SSqlInfo info1 = doGenerateAST("select block_dist() as a1 from `t.1abc`"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 1); -// -// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 1); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BINARY); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -// // ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); -// // ASSERT_EQ(p1->base.colInfo.colId, 0); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_UDC); -// ASSERT_STRCASEEQ(p1->base.token, "block_dist()"); -// ASSERT_EQ(p1->base.interBytes, 0); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 1); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} +TEST(testCase, validateAST_test) { + SSqlInfo info1 = doGenerateAST("select a a1111, a+b + 22, tbname from `t.1abc` where tsexprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 3); + + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_INT); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1111"); + ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); + ASSERT_EQ(p1->base.pColumns->info.colId, 1); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); + ASSERT_STRCASEEQ(p1->base.token, "a1111"); + + ASSERT_EQ(taosArrayGetSize(pExprList), 3); + + SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); + ASSERT_EQ(p2->base.pColumns->uid, 110); + ASSERT_EQ(p2->base.numOfParams, 1); // it is the serialized binary string of expression. + ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p2->base.resSchema.name, "a+b + 22"); + + // ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a"); + // ASSERT_EQ(p1->base.colInfo.colId, 1); + // ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL); + ASSERT_STRCASEEQ(p2->base.token, "a+b + 22"); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, function_Test) { + SSqlInfo info1 = doGenerateAST("select count(a) from `t.1abc`"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 1); + + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a)"); + ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); + ASSERT_EQ(p1->base.pColumns->info.colId, 1); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); + ASSERT_STRCASEEQ(p1->base.token, "count(a)"); + ASSERT_EQ(p1->base.interBytes, 8); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, function_Test2) { + SSqlInfo info1 = doGenerateAST("select count(a) abc from `t.1abc`"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 1); + + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "abc"); + ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.a"); + ASSERT_EQ(p1->base.pColumns->info.colId, 1); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); + ASSERT_STRCASEEQ(p1->base.token, "count(a)"); + ASSERT_EQ(p1->base.interBytes, 8); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, function_Test3) { + SSqlInfo info1 = doGenerateAST("select first(*) from `t.1abc`"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 6); + + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "first(ts)"); + ASSERT_STRCASEEQ(p1->base.pColumns->name, "t.1abc.ts"); + ASSERT_EQ(p1->base.pColumns->info.colId, 0); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); + ASSERT_STRCASEEQ(p1->base.token, "first(ts)"); + ASSERT_EQ(p1->base.interBytes, 24); + + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 6); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, function_Test4) { + SSqlInfo info1 = doGenerateAST("select block_dist() as a1 from `t.1abc`"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 1); + + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 1); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BINARY); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); + // ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts"); + // ASSERT_EQ(p1->base.colInfo.colId, 0); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_UDC); + ASSERT_STRCASEEQ(p1->base.token, "block_dist()"); + ASSERT_EQ(p1->base.interBytes, 0); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 1); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} TEST(testCase, function_Test5) { // todo select concat(concat(a, b), concat(b, a)) from `t.1abc`; @@ -372,7 +372,7 @@ TEST(testCase, function_Test5) { ASSERT_EQ(ret, 0); SArray* pExprList = pQueryInfo->exprList[0]; - ASSERT_EQ(taosArrayGetSize(pExprList), 3); + ASSERT_EQ(taosArrayGetSize(pExprList), 1); SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); ASSERT_EQ(p1->base.numOfCols, 2); @@ -395,249 +395,253 @@ TEST(testCase, function_Test5) { destroySqlInfo(&info1); } -//TEST(testCase, function_Test10) { -// sqlCheck("select c from `t.1abc`", true); -// sqlCheck("select length(c) from `t.1abc`", true); -// sqlCheck("select sum(length(a+b)) from `t.1abc`", true); -// sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); -// sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); -// sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); -// sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); -// sqlCheck("select cov(a, b) from `t.1abc`", true); -// // sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); -// // sqlCheck("select length(length(length(a))) from `t.1abc`", true); -//} -// -//TEST(testCase, function_Test6) { -// SSqlInfo info1 = doGenerateAST( -// "select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_EQ(ret, 0); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 5); -// -// SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); -// ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); -// ASSERT_EQ(p1->base.interBytes, 16); -// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); -// ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "sum"); -// ASSERT_EQ(p1->pExpr->_function.num, 1); -// -// tExprNode* pParam = p1->pExpr->_function.pChild[0]; -// -// ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); -// ASSERT_STREQ(pParam->pSchema->name, "t.1abc.a+b"); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); -// -// SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); -// ASSERT_EQ(p2->base.pColumns->uid, 110); -// ASSERT_EQ(p2->base.numOfParams, 0); -// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); -// -// ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_TMP); -// ASSERT_STREQ(p2->base.pColumns->name, "t.1abc.b*a"); -// -// ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); -// ASSERT_EQ(p2->base.interBytes, 24); -// ASSERT_EQ(p2->pExpr->nodeType, TEXPR_FUNCTION_NODE); -// ASSERT_STRCASEEQ(p2->pExpr->_function.functionName, "first"); -// ASSERT_EQ(p2->pExpr->_function.num, 1); -// ASSERT_EQ(p2->pExpr->_function.pChild[0]->nodeType, TEXPR_COL_NODE); -// ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} +TEST(testCase, function_Test10) { + sqlCheck("select c from `t.1abc`", true); + sqlCheck("select length(c) from `t.1abc`", true); + sqlCheck("select sum(length(a+b)) from `t.1abc`", true); + sqlCheck("select sum(sum(a+b)) from `t.1abc`", false); + sqlCheck("select sum(length(a) + length(b)) from `t.1abc`", true); + sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); + sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); + sqlCheck("select cov(a, b) from `t.1abc`", true); + // sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); + // sqlCheck("select length(length(length(a))) from `t.1abc`", true); +} -// TEST(testCase, function_Test7) { -// SSqlInfo info1 = doGenerateAST("select count(a+b),count(1) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_EQ(ret, 0); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 2); -// -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 0); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a+b)"); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "count(a+b)"); -// ASSERT_EQ(p1->base.interBytes, 8); -// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); -// ASSERT_STREQ(p1->pExpr->_function.functionName, "count"); -// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); -// -// tExprNode* pParam = p1->pExpr->_node.pLeft; -// -// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); -// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_ADD); -// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_COL_NODE); -// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_COL_NODE); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// -// TEST(testCase, function_Test8) { -// SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_EQ(ret, 0); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 2); -// -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 1); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); -// ASSERT_EQ(p1->base.interBytes, 16); -// -// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_UNARYEXPR_NODE); -// ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "top"); -// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); -// -// tExprNode* pParam = p1->pExpr->_node.pLeft; -// -// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); +TEST(testCase, function_Test6) { + SSqlInfo info1 = doGenerateAST( + "select sum(a+b) as a1, first(b*a), count(b+b), count(1), count(42.1) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 5); + + SExprInfo* p1 = (SExprInfo*)taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)"); + ASSERT_EQ(p1->base.interBytes, 16); + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "sum"); + ASSERT_EQ(p1->pExpr->_function.num, 1); + + tExprNode* pParam = p1->pExpr->_function.pChild[0]; + + ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); + ASSERT_STREQ(pParam->pSchema->name, "t.1abc.a+b"); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 5); + + SExprInfo* p2 = (SExprInfo*)taosArrayGetP(pExprList, 1); + ASSERT_EQ(p2->base.pColumns->uid, 110); + ASSERT_EQ(p2->base.numOfParams, 0); + ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p2->base.resSchema.name, "first(b*a)"); + + ASSERT_EQ(p2->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STREQ(p2->base.pColumns->name, "t.1abc.b*a"); + + ASSERT_STRCASEEQ(p2->base.token, "first(b*a)"); + ASSERT_EQ(p2->base.interBytes, 24); + ASSERT_EQ(p2->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STRCASEEQ(p2->pExpr->_function.functionName, "first"); + ASSERT_EQ(p2->pExpr->_function.num, 1); + ASSERT_EQ(p2->pExpr->_function.pChild[0]->nodeType, TEXPR_COL_NODE); + ASSERT_STREQ(p2->pExpr->_function.pChild[0]->pSchema->name, "t.1abc.b*a"); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + + TEST(testCase, function_Test7) { + SSqlInfo info1 = doGenerateAST("select count(a+b),count(1) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 2); + + SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 0); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a+b)"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STRCASEEQ(p1->base.token, "count(a+b)"); + ASSERT_EQ(p1->base.interBytes, 8); + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STREQ(p1->pExpr->_function.functionName, "count"); + + tExprNode* pParam = p1->pExpr->_function.pChild[0]; + ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); + + SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pQueryInfo->exprList[1], 0); + ASSERT_EQ(p2->pExpr->nodeType, TEXPR_BINARYEXPR_NODE); + + ASSERT_EQ(p2->pExpr->_node.optr, TSDB_BINARY_OP_ADD); + ASSERT_EQ(p2->pExpr->_node.pLeft->nodeType, TEXPR_COL_NODE); + ASSERT_EQ(p2->pExpr->_node.pRight->nodeType, TEXPR_COL_NODE); + + ASSERT_EQ(pParam->pSchema->colId, p2->base.resSchema.colId); + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + + TEST(testCase, function_Test8) { + SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 2); + + SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 1); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); + ASSERT_EQ(p1->base.interBytes, 16); + + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STRCASEEQ(p1->pExpr->_function.functionName, "top"); + ASSERT_TRUE(p1->pExpr->_function.num == 1); + + tExprNode* pParam = p1->pExpr->_function.pChild[0]; + + ASSERT_EQ(pParam->nodeType, TSDB_COL_TMP); // ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); // ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); // ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} -// -// TEST(testCase, invalid_sql_Test) { -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlInfo info1 = doGenerateAST("select count(k) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_NE(ret, 0); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -////=============================================================================================================== -// info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_NE(ret, 0); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} \ No newline at end of file + + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + + TEST(testCase, invalid_sql_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlInfo info1 = doGenerateAST("select count(k) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_NE(ret, 0); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +//=============================================================================================================== + info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_NE(ret, 0); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} \ No newline at end of file From f0b71c971a5f9db76a0ed5eb1e163b4f4bdb8907 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 17:57:12 +0800 Subject: [PATCH 13/26] [td-10564]fix bug in parser --- source/libs/parser/src/astValidate.c | 4 ++++ source/libs/parser/test/parserTests.cpp | 27 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 4098fa5392..9c3537e261 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3026,6 +3026,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; (*pExpr)->_function.pChild = p; + (*pExpr)->_function.num = num; strncpy((*pExpr)->_function.functionName, pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); return TSDB_CODE_SUCCESS; } else { @@ -3155,6 +3156,9 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } } } + + // scalar op aggregate check + } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 79bdb561b7..00a31dc89d 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -404,6 +404,8 @@ TEST(testCase, function_Test10) { sqlCheck("select length(sum(a) + sum(b)) + length(sum(a) + sum(b)) from `t.1abc`", true); sqlCheck("select sum(length(sum(a))) from `t.1abc`", true); sqlCheck("select cov(a, b) from `t.1abc`", true); + sqlCheck("select sum(length(a) + count(b)) from `t.1abc`", false); + // sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); // sqlCheck("select length(length(length(a))) from `t.1abc`", true); } @@ -582,7 +584,7 @@ TEST(testCase, function_Test6) { tExprNode* pParam = p1->pExpr->_function.pChild[0]; ASSERT_EQ(pParam->nodeType, TSDB_COL_TMP); -// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); +// ASSERT_EQ(pParam->.optr, TSDB_BINARY_OP_DIVIDE); // ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); // ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); @@ -592,6 +594,29 @@ TEST(testCase, function_Test6) { destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); destroySqlInfo(&info1); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + info1 = doGenerateAST("select sum(length(a)+length(b)) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); } TEST(testCase, invalid_sql_Test) { From 7c9d7600265275ebabd94b35fe0e1e53383838b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 23:24:11 +0800 Subject: [PATCH 14/26] [td-10564] Fix bug in create logic plan. --- include/common/taosmsg.h | 4 + include/libs/function/function.h | 2 +- include/libs/parser/parser.h | 2 + source/libs/function/src/tfunction.c | 7 +- source/libs/parser/inc/parserUtil.h | 2 - source/libs/parser/src/parserUtil.c | 2 +- source/libs/parser/test/plannerTest.cpp | 120 ++++++++++++------------ source/libs/planner/src/planner.c | 102 +++++++++++++------- 8 files changed, 141 insertions(+), 100 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index bd1964187d..ccc8d3023e 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -495,6 +495,10 @@ typedef struct SSessionWindow { int32_t primaryColId; // primary timestamp column } SSessionWindow; +typedef struct SStateWindow { + int32_t columnId; +} SStateWindow; + typedef struct { SMsgHead head; char version[TSDB_VERSION_LEN]; diff --git a/include/libs/function/function.h b/include/libs/function/function.h index f4c1fbbbc0..f7d1fb9e45 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -259,7 +259,7 @@ int32_t qIsBuiltinFunction(const char* name, int32_t len, bool* scalarFunction); bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); -const char* qGetFunctionName(int32_t functionId); +bool qIsAggregateFunction(const char* functionName); tExprNode* exprTreeFromBinary(const void* data, size_t size); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index c57d28ebf4..64706a922b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -77,6 +77,7 @@ typedef struct SQueryStmtInfo { STimeWindow window; // the whole query time window SInterval interval; // tumble time window SSessionWindow sessionWindow; // session time window + SStateWindow stateWindow; // state window query SGroupbyExpr groupbyExpr; // groupby tags info SArray * colList; // SArray SFieldInfo fieldsInfo; @@ -180,6 +181,7 @@ STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); int32_t getNewResColId(); +void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); #ifdef __cplusplus } diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index c47f3a249a..2e4a4a058e 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -46,10 +46,15 @@ bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* funct return true; } -const char* qGetFunctionName(int32_t functionId) { +bool qIsAggregateFunction(const char* functionName) { + assert(functionName != NULL); + bool scalefunc = false; + qIsBuiltinFunction(functionName, strlen(functionName), &scalefunc); + return !scalefunc; } + SAggFunctionInfo* qGetFunctionInfo(const char* name, int32_t len) { pthread_once(&functionHashTableInit, doInitFunctionHashTable); diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index f567f6553b..c57ea905e9 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -41,8 +41,6 @@ SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* nam void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); -SSourceParam addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); - SInternalField* insertFieldInfo(SFieldInfo* pFieldInfo, int32_t index, SSchema* field); int32_t getNumOfFields(SFieldInfo* pFieldInfo); SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index a3a6e58785..ecb707e582 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -544,7 +544,7 @@ SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSc return c; } -SSourceParam addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn) { +void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn) { assert(pSourceParam != NULL); pSourceParam->num += 1; diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 6e091bf8f4..7eb45e92d2 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include #include #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -66,65 +67,60 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { } } -//TEST(testCase, planner_test) { -// SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); -// ASSERT_EQ(info1.valid, true); -// -// char msg[128] = {0}; -// SMsgBuf buf; -// buf.len = 128; -// buf.buf = msg; -// -// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); -// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); -// ASSERT_EQ(code, 0); -// -// SMetaReq req = {0}; -// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); -// ASSERT_EQ(ret, 0); -// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); -// -// SQueryStmtInfo* pQueryInfo = createQueryInfo(); -// setTableMetaInfo(pQueryInfo, &req); -// -// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); -// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); -// ASSERT_EQ(ret, 0); -// -// SArray* pExprList = pQueryInfo->exprList[0]; -// ASSERT_EQ(taosArrayGetSize(pExprList), 2); -// -// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); -// ASSERT_EQ(p1->base.pColumns->uid, 110); -// ASSERT_EQ(p1->base.numOfParams, 1); -// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); -// ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); -// ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_NORMAL); -// ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); -// ASSERT_EQ(p1->base.interBytes, 16); -// -// ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); -// ASSERT_EQ(p1->pExpr->_function.functionId, FUNCTION_TOP); -// ASSERT_TRUE(p1->pExpr->_node.pRight == NULL); -// -// tExprNode* pParam = p1->pExpr->_node.pLeft; -// -// ASSERT_EQ(pParam->nodeType, TEXPR_BINARYEXPR_NODE); -// ASSERT_EQ(pParam->_node.optr, TSDB_BINARY_OP_DIVIDE); -// ASSERT_EQ(pParam->_node.pLeft->nodeType, TEXPR_BINARYEXPR_NODE); -// ASSERT_EQ(pParam->_node.pRight->nodeType, TEXPR_VALUE_NODE); -// -// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); -// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); -// -// struct SQueryPlanNode* n = nullptr; -// code = qCreateQueryPlan(pQueryInfo, &n); -// -// char* str = NULL; -// qQueryPlanToString(n, &str); -// printf("%s\n", str); -// -// destroyQueryInfo(pQueryInfo); -// qParserClearupMetaRequestInfo(&req); -// destroySqlInfo(&info1); -//} \ No newline at end of file +TEST(testCase, planner_test) { + SSqlInfo info1 = doGenerateAST("select top(a*b / 99, 20) from `t.1abc` interval(10s, 1s)"); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + SArray* pExprList = pQueryInfo->exprList[0]; + ASSERT_EQ(taosArrayGetSize(pExprList), 2); + + SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 1); + ASSERT_EQ(p1->base.pColumns->uid, 110); + ASSERT_EQ(p1->base.numOfParams, 1); + ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STRCASEEQ(p1->base.resSchema.name, "top(a*b / 99, 20)"); + ASSERT_EQ(p1->base.pColumns->flag, TSDB_COL_TMP); + ASSERT_STRCASEEQ(p1->base.token, "top(a*b / 99, 20)"); + ASSERT_EQ(p1->base.interBytes, 16); + + ASSERT_EQ(p1->pExpr->nodeType, TEXPR_FUNCTION_NODE); + ASSERT_STREQ(p1->pExpr->_function.functionName, "top"); + + tExprNode* pParam = p1->pExpr->_function.pChild[0]; + + ASSERT_EQ(pParam->nodeType, TEXPR_COL_NODE); + ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3); + ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); + + struct SQueryPlanNode* n = nullptr; + code = qCreateQueryPlan(pQueryInfo, &n); + + char* str = NULL; + qQueryPlanToString(n, &str); + printf("%s\n", str); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} \ No newline at end of file diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 81404d74ca..0621a90798 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "os.h" -#include "plannerInt.h" -#include "parser.h" #include "function.h" +#include "os.h" +#include "parser.h" +#include "plannerInt.h" #define QNODE_TAGSCAN 1 #define QNODE_TABLESCAN 2 @@ -30,7 +30,8 @@ #define QNODE_UNIONALL 10 #define QNODE_TIMEWINDOW 11 #define QNODE_SESSIONWINDOW 12 -#define QNODE_FILL 13 +#define QNODE_STATEWINDOW 13 +#define QNODE_FILL 14 typedef struct SFillEssInfo { int32_t fillType; // fill type @@ -161,7 +162,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, SArray* pExprs, SArray* tableCols) { if (pQueryInfo->info.onlyTagQuery) { - int32_t num = (int32_t) taosArrayGetSize(pExprs); + int32_t num = (int32_t) taosArrayGetSize(pExprs); SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL); if (pQueryInfo->info.distinct) { @@ -178,20 +179,19 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); } else { + STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); + // table source column projection, generate the projection expr int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - - STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); - for (int32_t i = 0; i < numOfCols; ++i) { SColumn* pCol = taosArrayGetP(tableCols, i); - SColumnIndex index = {.tableIndex = 0, /*.columnIndex = pCol->columnIndex*/}; + SSchema* pSchema = getOneColumnSchema(pTableMetaInfo1->pTableMeta, i); - SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, i); - SSchema resultSchema = *pSchema; + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, pCol); - SExprInfo* p = NULL;//createExprInfo(pTableMetaInfo1, FUNCTION_PRJ, &index, NULL, &resultSchema, 0); + SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, pSchema, 0); pExpr[i] = p; } @@ -202,33 +202,69 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe return pNode; } -static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info, - SArray* pExprs) { +static int32_t getFunctionLevel(SQueryStmtInfo* pQueryInfo) { + int32_t n = 10; + + int32_t level = 0; + for(int32_t i = 0; i < n; ++i) { + SArray* pList = pQueryInfo->exprList[i]; + if (taosArrayGetSize(pList) > 0) { + level += 1; + } + } + + return level; +} + +static SQueryPlanNode* createOneQueryPlanNode(SArray* p, SQueryPlanNode* pNode, SExprInfo* pExpr, SQueryTableInfo* info) { + if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE) { + bool aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); + if (aggregateFunc) { + int32_t numOfOutput = (int32_t)taosArrayGetSize(p); + return createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, numOfOutput, info, NULL); + } else { + int32_t numOfOutput = (int32_t)taosArrayGetSize(p); + return createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, numOfOutput, info, NULL); + } + } else { + int32_t numOfOutput = (int32_t)taosArrayGetSize(p); + return createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, numOfOutput, info, NULL); + } +} + +static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info, SArray** pExprs) { // check for aggregation size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); - if (pQueryInfo->interval.interval > 0) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); + int32_t level = getFunctionLevel(pQueryInfo); + for(int32_t i = level - 1; i >= 0; --i) { + SArray* p = pQueryInfo->exprList[i]; + SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->interval); - if (numOfGroupCols != 0) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->groupbyExpr); + if (i == 0) { + if (pQueryInfo->interval.interval > 0) { + int32_t numOfOutput = (int32_t)taosArrayGetSize(p); + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, numOfOutput, info, &pQueryInfo->interval); + } else if (pQueryInfo->sessionWindow.gap > 0) { + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); + } else if (pQueryInfo->stateWindow.columnId > 0) { + pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, NULL, 0, info, NULL); + } else { + pNode = createOneQueryPlanNode(p, pNode, pExpr, info); + } + } else { + pNode = createOneQueryPlanNode(p, pNode, pExpr, info); } - } else if (numOfGroupCols > 0) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, - &pQueryInfo->groupbyExpr); - } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); - } else if (pQueryInfo->info.simpleAgg) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, pExprs->pData, numOfOutput, info, NULL); } - if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->info.arithmeticOnAgg) { + if (numOfGroupCols != 0) { + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, NULL, 0, info, &pQueryInfo->groupbyExpr); + } + + if (pQueryInfo->havingFieldNum > 0) { // int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); -// pNode = -// createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); +// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, +// NULL); } if (pQueryInfo->fillType != TSDB_FILL_NONE) { @@ -314,7 +350,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { // 3. add the join node here SQueryTableInfo info = {0}; - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList); + int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, pQueryInfo->exprList[0]->pData, num, &info, NULL); @@ -324,7 +360,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { taosArrayPush(upstream, &pNode); } else { // only one table, normal query process STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryPlanNode* pNode = doCreateQueryPlanForOneTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList); + SQueryPlanNode* pNode = doCreateQueryPlanForOneTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); upstream = taosArrayInit(5, POINTER_BYTES); taosArrayPush(upstream, &pNode); } From bdcaeff12b50a852ebc60b5eb53bb3e87c39b1f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 8 Nov 2021 23:33:14 +0800 Subject: [PATCH 15/26] [td-10564] Fix bug in parser sql. --- source/libs/parser/src/astValidate.c | 11 +++++++++-- source/libs/parser/test/parserTests.cpp | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 9c3537e261..545f2dd838 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3137,7 +3137,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } } else { - *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode)); + *pExpr = (tExprNode*)calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TEXPR_BINARYEXPR_NODE; (*pExpr)->_node.pLeft = pLeft; @@ -3158,9 +3158,16 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } // scalar op aggregate check + if (pLeft->nodeType == TEXPR_FUNCTION_NODE && pRight->nodeType != TEXPR_FUNCTION_NODE) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); + } + if (pLeft->nodeType == TEXPR_FUNCTION_NODE && pRight->nodeType == TEXPR_FUNCTION_NODE) { + if (qIsAggregateFunction(pLeft->_function.functionName) != qIsAggregateFunction(pRight->_function.functionName)) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); + } + } } - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 00a31dc89d..45b8a3627b 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -406,8 +406,8 @@ TEST(testCase, function_Test10) { sqlCheck("select cov(a, b) from `t.1abc`", true); sqlCheck("select sum(length(a) + count(b)) from `t.1abc`", false); - // sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); - // sqlCheck("select length(length(length(a))) from `t.1abc`", true); + sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); + sqlCheck("select length(length(length(a))) from `t.1abc`", true); } TEST(testCase, function_Test6) { From a7058d818a2286d3b63c5ecc5e19cdd1c54a4bb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Nov 2021 11:08:25 +0800 Subject: [PATCH 16/26] [td-10564] Fix bug in parse sql. --- source/libs/parser/inc/sql.y | 3 -- source/libs/parser/src/astValidate.c | 10 ++++-- source/libs/parser/test/parserTests.cpp | 6 ++-- source/libs/parser/test/plannerTest.cpp | 45 +++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 0296b1cde9..a7577c4e4b 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -30,7 +30,6 @@ #include "tmsgtype.h" #include "ttoken.h" #include "ttokendef.h" -//#include "tutil.h" #include "tvariant.h" } @@ -784,10 +783,8 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { X.n += F.n; - toTSDBType(A.type); SArray* K = tListItemAppendToken(NULL, &A, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 545f2dd838..7fbc7e2782 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -2017,12 +2017,18 @@ int32_t extractFunctionParameterInfo(SQueryStmtInfo* pQueryInfo, int32_t tokenId static int32_t checkForkParam(tSqlExpr* pSqlExpr, size_t k, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid parameters"; + SArray* pParamList = pSqlExpr->Expr.paramList; + if (k == 0) { - if (pSqlExpr->Expr.paramList != NULL && taosArrayGetSize(pSqlExpr->Expr.paramList) != 0) { + if (pParamList != NULL && taosArrayGetSize(pParamList) != 0) { return buildInvalidOperationMsg(pMsgBuf, msg1); } + } else if (k == 1) { + if (!(pParamList == NULL || taosArrayGetSize(pParamList) == k)) { + return buildInvalidOperationMsg(pMsgBuf, msg1);; + } } else { - if (pSqlExpr->Expr.paramList == NULL || taosArrayGetSize(pSqlExpr->Expr.paramList) != k) { + if (pParamList != NULL && taosArrayGetSize(pParamList) != k) { return buildInvalidOperationMsg(pMsgBuf, msg1); } } diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 45b8a3627b..932613301b 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -406,8 +406,10 @@ TEST(testCase, function_Test10) { sqlCheck("select cov(a, b) from `t.1abc`", true); sqlCheck("select sum(length(a) + count(b)) from `t.1abc`", false); - sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); - sqlCheck("select length(length(length(a))) from `t.1abc`", true); + sqlCheck("select concat(concat(a,b), concat(a,b)) from `t.1abc`", true); + sqlCheck("select length(length(length(a))) from `t.1abc`", true); + sqlCheck("select count() from `t.1abc`", false); + sqlCheck("select block_dist() from `t.1abc`", true); } TEST(testCase, function_Test6) { diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 7eb45e92d2..5d3a6a42a6 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -64,6 +64,44 @@ void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) { setSchema(&pSchema[2], TSDB_DATA_TYPE_DOUBLE, 8, "b", 2); setSchema(&pSchema[3], TSDB_DATA_TYPE_DOUBLE, 8, "col", 3); +} + +void generateLogicplan(const char* sql) { + SSqlInfo info1 = doGenerateAST(sql); + ASSERT_EQ(info1.valid, true); + + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); + ASSERT_EQ(code, 0); + + SMetaReq req = {0}; + int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128); + ASSERT_EQ(ret, 0); + ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); + + SQueryStmtInfo* pQueryInfo = createQueryInfo(); + setTableMetaInfo(pQueryInfo, &req); + + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); + ASSERT_EQ(ret, 0); + + struct SQueryPlanNode* n = nullptr; + code = qCreateQueryPlan(pQueryInfo, &n); + + char* str = NULL; + qQueryPlanToString(n, &str); + printf("%s\n", str); + + destroyQueryInfo(pQueryInfo); + qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); + } } @@ -123,4 +161,11 @@ TEST(testCase, planner_test) { destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); destroySqlInfo(&info1); +} + +TEST(testCase, displayPlan) { + generateLogicplan("select count(*) from `t.1abc`"); + generateLogicplan("select count(*) from `t.1abc` group by a"); + generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); + generateLogicplan("select count(*),sum(a),avg(b),min(a+b) from `t.1abc`"); } \ No newline at end of file From 3c8a25aa106329a139aa64c1f3eee3bc9825dbce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Nov 2021 11:17:37 +0800 Subject: [PATCH 17/26] [td-10564] refactor --- source/libs/parser/test/plannerTest.cpp | 10 ++++++++++ source/libs/planner/src/planner.c | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 5d3a6a42a6..89e4166070 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -168,4 +168,14 @@ TEST(testCase, displayPlan) { generateLogicplan("select count(*) from `t.1abc` group by a"); generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); generateLogicplan("select count(*),sum(a),avg(b),min(a+b) from `t.1abc`"); + + // order by + group by column + limit offset + fill + + + // join + + + // union + + } \ No newline at end of file diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 0621a90798..00693892e9 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -154,6 +154,8 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); break; } + default: + assert(0); } return pNode; @@ -257,14 +259,14 @@ static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryIn } } + // group by column not by tag if (numOfGroupCols != 0) { pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, NULL, 0, info, &pQueryInfo->groupbyExpr); } if (pQueryInfo->havingFieldNum > 0) { // int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); -// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, -// NULL); +// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); } if (pQueryInfo->fillType != TSDB_FILL_NONE) { From addb829f62802f86524762336bb4e89d8ccc29a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 9 Nov 2021 13:43:50 +0800 Subject: [PATCH 18/26] [td-10564] merge 3.0 branch. --- source/libs/parser/src/parserUtil.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index ecb707e582..17dbde39da 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -609,8 +609,9 @@ void fieldInfoUpdateOffset(SQueryStmtInfo* pQueryInfo) { int32_t offset = 0; size_t numOfExprs = getNumOfExprs(pQueryInfo); + SArray* pList = getCurrentExprList(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, i); + SExprInfo* p = taosArrayGetP(pList, i); // p->base.offset = offset; offset += p->base.resSchema.bytes; @@ -1216,13 +1217,13 @@ int32_t queryInfoCopy(SQueryStmtInfo* pQueryInfo, const SQueryStmtInfo* pSrc) { memcpy(pQueryInfo->fillVal, pSrc->fillVal, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t)); } - if (copyAllExprInfo(pQueryInfo->exprList, pSrc->exprList, true) != 0) { + if (copyAllExprInfo(pQueryInfo->exprList[0], pSrc->exprList[0], true) != 0) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } columnListCopyAll(pQueryInfo->colList, pSrc->colList); - copyFieldInfo(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList); + copyFieldInfo(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList[0]); for(int32_t i = 0; i < pSrc->numOfTables; ++i) { STableMetaInfo* p1 = getMetaInfo((SQueryStmtInfo*) pSrc, i); From e1d2a12def1c4de50f925e666308b452db613396 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 10 Nov 2021 15:52:26 +0800 Subject: [PATCH 19/26] test for CI --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ef40b113ac..fde917b64b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,7 +142,7 @@ pipeline { } } // stage('Parallel test stage') { - // //only build pr + // options { skipDefaultCheckout() } // when { // allOf{ From 10095a4ae29714a3d240e5628516d0be024e09ac Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Wed, 10 Nov 2021 15:54:51 +0800 Subject: [PATCH 20/26] Revert "test for CI" --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fde917b64b..ef40b113ac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,7 +142,7 @@ pipeline { } } // stage('Parallel test stage') { - + // //only build pr // options { skipDefaultCheckout() } // when { // allOf{ From a4764c36d6574b1efed7cc05992089f9c1aa9413 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 10 Nov 2021 15:58:33 +0800 Subject: [PATCH 21/26] skip default checkout --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fde917b64b..c19a720d49 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,7 +142,7 @@ pipeline { } } // stage('Parallel test stage') { - + // skip defaultCheckout // options { skipDefaultCheckout() } // when { // allOf{ From 7cdf65102c73ec68c948df15c3cdc4a5019ef437 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 10 Nov 2021 16:16:24 +0800 Subject: [PATCH 22/26] test for CI --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c19a720d49..a822c5b0e1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,8 +4,8 @@ import jenkins.model.CauseOfInterruption node { } -def skipbuild=0 -def win_stop=0 +// def skipbuild=0 +// def win_stop=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME From 94b2e2ddc1c0abb65855bd6c9181c6c7f9725c5c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Nov 2021 16:56:13 +0800 Subject: [PATCH 23/26] [td-10564] Fix bug in generating sql plan. --- include/common/common.h | 4 +- include/libs/parser/parser.h | 1 + source/libs/executor/src/executorimpl.c | 3 - source/libs/parser/inc/parserUtil.h | 3 +- source/libs/parser/src/astValidate.c | 304 +++++++++++++++--------- source/libs/parser/src/parserUtil.c | 6 +- source/libs/parser/test/plannerTest.cpp | 23 +- source/libs/planner/src/planner.c | 143 ++++++++--- src/query/src/qPlan.c | 4 +- 9 files changed, 335 insertions(+), 156 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index 8f9b595d7a..be1a85e0e8 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -86,10 +86,8 @@ typedef struct SOrder { } SOrder; typedef struct SGroupbyExpr { - int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc + bool groupbyTag; // group by tag or column } SGroupbyExpr; // the structure for sql function in select clause diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 64706a922b..9e6a033f10 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -179,6 +179,7 @@ int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deep STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); +SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); int32_t getNewResColId(); void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* pColumn); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b54d53bcea..3b22fd3fea 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7851,9 +7851,6 @@ SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pCo return NULL; } - pGroupbyExpr->orderType = pQueryMsg->orderType; - pGroupbyExpr->orderIndex = pQueryMsg->orderByIdx; - pGroupbyExpr->columnInfo = taosArrayInit(pQueryMsg->numOfGroupCols, sizeof(SColIndex)); for(int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { taosArrayPush(pGroupbyExpr->columnInfo, &pColIndex[i]); diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index c57ea905e9..12ffe696c1 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -37,7 +37,6 @@ extern "C" { (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE)) TAOS_FIELD createField(const SSchema* pSchema); -SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); void setColumn(SColumn* pColumn, uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); SColumn createColumn(uint64_t uid, const char* tableName, int8_t flag, const SSchema* pSchema); @@ -54,7 +53,7 @@ STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo); void columnListCopyAll(SArray* dst, const SArray* src); SColumn* columnListInsert(SArray* pColumnList, uint64_t uid, SSchema* pSchema, int32_t flag); -SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid); +SColumn* insertPrimaryTsColumn(SArray* pColumnList, const char* colName, uint64_t tableUid); void cleanupTagCond(STagCond* pTagCond); void cleanupColumnCond(SArray** pCond); diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 7fbc7e2782..0aa7282e5c 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -547,30 +547,21 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, SMsgBuf* groupbyTag = true; - int32_t relIndex = index.columnIndex; - if (index.columnIndex != TSDB_TBNAME_COLUMN_INDEX) { - relIndex -= getNumOfColumns(pTableMeta); - } + SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_TAG, pSchema); + taosArrayPush(pGroupExpr->columnInfo, &c); - SColIndex colIndex = { .colIndex = relIndex, .flag = TSDB_COL_TAG, .colId = pSchema->colId, }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - - index.columnIndex = relIndex; - columnListInsert(pTableMetaInfo->tagColList, pTableMeta->uid, pSchema, colIndex.flag); + columnListInsert(pTableMetaInfo->tagColList, pTableMeta->uid, pSchema, TSDB_COL_TAG); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by if (pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { return buildInvalidOperationMsg(pMsgBuf, msg5); } + SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, TSDB_COL_NORMAL, pSchema); + taosArrayPush(pGroupExpr->columnInfo, &c); + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; - strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name)); - - taosArrayPush(pGroupExpr->columnInfo, &colIndex); - numOfGroupbyCols++; pQueryInfo->info.groupbyColumn = true; } @@ -589,8 +580,7 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, SMsgBuf* } } - pGroupExpr->orderType = TSDB_ORDER_ASC; - pGroupExpr->tableIndex = tableIndex; + pGroupExpr->groupbyTag = groupbyTag; return TSDB_CODE_SUCCESS; } @@ -860,7 +850,6 @@ int32_t validateStateWindowNode(SQueryStmtInfo *pQueryInfo, SWindowStateVal* pWi //TODO use group by routine? state window query not support stable query. taosArrayPush(pGroupExpr->columnInfo, &colIndex); - pGroupExpr->orderType = TSDB_ORDER_ASC; pQueryInfo->info.stateWindow = true; return TSDB_CODE_SUCCESS; @@ -1732,7 +1721,8 @@ SExprInfo* doAddOneExprInfo(SQueryStmtInfo* pQueryInfo, const char* funcName, SS } if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { - insertPrimaryTsColumn(pQueryInfo->colList, uid); + char* colName = pTableMetaInfo->pTableMeta->schema[0].name; + insertPrimaryTsColumn(pQueryInfo->colList, colName, uid); } } @@ -2809,7 +2799,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* // add the primary timestamp column even though it is not required by user STableMeta* pTableMeta = getMetaInfo(pQueryInfo, index.tableIndex)->pTableMeta; if (pTableMeta->tableType != TSDB_TEMP_TABLE) { - insertPrimaryTsColumn(pQueryInfo->colList, pTableMeta->uid); + insertPrimaryTsColumn(pQueryInfo->colList, pTableMeta->schema[0].name, pTableMeta->uid); } } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { //constant value column SColumnIndex index = createConstantColumnIndex(&pQueryInfo->udColumnId); @@ -2848,7 +2838,8 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem* // add the primary timestamp column even though it is not required by user STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); if (!UTIL_TABLE_IS_TMP_TABLE(pTableMetaInfo)) { - insertPrimaryTsColumn(pQueryInfo->colList, pTableMetaInfo->pTableMeta->uid); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + insertPrimaryTsColumn(pQueryInfo->colList, pTableMeta->schema[0].name, pTableMeta->uid); } } else { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -2967,7 +2958,143 @@ static uint64_t findTmpSourceColumnInNextLevel(SQueryStmtInfo* pQueryInfo, tExpr return uid; } -int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, bool* keepTableCols, SMsgBuf* pMsgBuf) { +static tExprNode* doCreateColumnNode(SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, bool keepTableCols, SArray* pCols) { + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + tExprNode* pExpr = calloc(1, sizeof(tExprNode)); + pExpr->nodeType = TEXPR_COL_NODE; + + pExpr->pSchema = calloc(1, sizeof(SSchema)); + SSchema* pSchema = getOneColumnSchema(pTableMeta, pIndex->columnIndex); + *(SSchema*)(pExpr->pSchema) = *pSchema; + + if (keepTableCols) { + SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, pIndex->type, pExpr->pSchema); + taosArrayPush(pCols, &c); + } + + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); + SSchema* pTsSchema = getOneColumnSchema(pTableMeta, 0); + insertPrimaryTsColumn(pQueryInfo->colList, pTsSchema->name, pTableMeta->uid); + return pExpr; +} + +static int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SMsgBuf* pMsgBuf); + +static int32_t doProcessFunctionLeafNodeParam(SQueryStmtInfo* pQueryInfo, int32_t* num, tExprNode** p, SArray* pCols, + bool* keepTableCols, const tSqlExpr* pSqlExpr, SMsgBuf* pMsgBuf) { + SArray* pParamList = pSqlExpr->Expr.paramList; + if (pParamList != NULL) { + *num = taosArrayGetSize(pParamList); + p = calloc((*num), POINTER_BYTES); + + for (int32_t i = 0; i < (*num); ++i) { + tSqlExprItem* pItem = taosArrayGet(pParamList, i); + + int32_t ret = validateSqlExpr(pItem->pNode, pQueryInfo, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + int32_t code = sqlExprToExprNode(&p[i], pItem->pNode, pQueryInfo, pCols, keepTableCols, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { // handle the case: count(*) + 22 + if (strncasecmp(pSqlExpr->Expr.operand.z, "count", pSqlExpr->Expr.operand.n) != 0) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); + } + + *num = 1; + p = calloc(*num, POINTER_BYTES); + + SColumnIndex index = {.type = TSDB_COL_NORMAL, .tableIndex = 0, .columnIndex = 0}; + p[0] = doCreateColumnNode(pQueryInfo, &index, *keepTableCols, pCols); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t doValidateExpr(SQueryStmtInfo *pQueryInfo, tSqlExpr* pFuncNode, tSqlExpr* pTableColumnNode, SMsgBuf* pMsgBuf) { + char token[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + strncpy(token, pFuncNode->Expr.operand.z, pFuncNode->Expr.operand.n); + bool isAgg = qIsAggregateFunction(token); + + // count(*) + column is a invalid expression. + if (isAgg) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); + } + return TSDB_CODE_SUCCESS; +} + +int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SMsgBuf* pMsgBuf) { + assert(pSqlExpr); + + if (pSqlExpr->type == SQL_NODE_EXPR) { + int32_t valid = validateSqlExpr(pSqlExpr->pLeft, pQueryInfo, pMsgBuf); + if (valid != TSDB_CODE_SUCCESS) { + return valid; + } + + valid = validateSqlExpr(pSqlExpr->pRight, pQueryInfo, pMsgBuf); + if (valid != TSDB_CODE_SUCCESS) { + return valid; + } + + tSqlExpr* pLeft = pSqlExpr->pLeft, *pRight = pSqlExpr->pRight; + if (pLeft->type == SQL_NODE_SQLFUNCTION && pRight->type == SQL_NODE_SQLFUNCTION) { + + char token[FUNCTIONS_NAME_MAX_LENGTH] = {0}; + tstrncpy(token, pLeft->Expr.operand.z, pLeft->Expr.operand.n); + bool agg1 = qIsAggregateFunction(token); + + tstrncpy(token, pRight->Expr.operand.z, pRight->Expr.operand.n); + bool agg2 = qIsAggregateFunction(token); + + if (agg1 != agg2) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); + } + } + + if (pLeft->type == SQL_NODE_SQLFUNCTION && pRight->type == SQL_NODE_TABLE_COLUMN) { + int32_t code = doValidateExpr(pQueryInfo, pLeft, pRight, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else if (pRight->type == SQL_NODE_SQLFUNCTION && pLeft->type == SQL_NODE_TABLE_COLUMN) { + int32_t code = doValidateExpr(pQueryInfo, pRight, pLeft, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + int32_t tokenId = pSqlExpr->tokenId; + if (pRight->type == SQL_NODE_VALUE && pRight->value.nType == TSDB_DATA_TYPE_DOUBLE && pRight->value.d == 0 && tokenId == TK_DIVIDE) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression (divided by 0)"); + } + + if (tokenId == TK_DIVIDE || tokenId == TK_TIMES || tokenId == TK_MINUS || tokenId == TK_PLUS || tokenId == TK_MODULES) { + if ((pRight->type == SQL_NODE_VALUE && pRight->value.nType == TSDB_DATA_TYPE_BINARY) || + (pLeft->type == SQL_NODE_VALUE && pLeft->value.nType == TSDB_DATA_TYPE_BINARY)) { + return buildInvalidOperationMsg(pMsgBuf, "invalid expression (string in arithmetic expression)"); + } + } + + } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + + int32_t ret = getColumnIndexByName(&pSqlExpr->columnName, pQueryInfo, &index, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryStmtInfo* pQueryInfo, SArray* pCols, bool* keepTableCols, SMsgBuf* pMsgBuf) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; @@ -2993,60 +3120,49 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt return TSDB_CODE_SUCCESS; } } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { - SArray* pParamList = pSqlExpr->Expr.paramList; + bool scalar = false; + int32_t functionId = qIsBuiltinFunction(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n, &scalar); + if (functionId < 0) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } - if (pParamList != NULL && taosArrayGetSize(pParamList) > 0) { - bool scalar = false; - int32_t functionId = qIsBuiltinFunction(pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n, &scalar); - if (functionId < 0) { + if (!scalar) { + pQueryInfo->exprListLevelIndex += 1; + } + + *keepTableCols = false; + + int32_t num = 0; + tExprNode** p = NULL; + int32_t code = doProcessFunctionLeafNodeParam(pQueryInfo, &num, p, pCols, keepTableCols, pSqlExpr, pMsgBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); + + if (scalar) { + printf("scalar function found! %s\n", pSqlExpr->exprToken.z); + + // Expression on the results of aggregation functions + *pExpr = calloc(1, sizeof(tExprNode)); + (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; + + (*pExpr)->_function.pChild = p; + (*pExpr)->_function.num = num; + strncpy((*pExpr)->_function.functionName, pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); + return TSDB_CODE_SUCCESS; + } else { + printf("agg function found, %s\n", pSqlExpr->exprToken.z); + tSqlExprItem item = {.pNode = (tSqlExpr*)pSqlExpr, .aliasName = NULL, .functionId = functionId}; + if (addAggExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } - if (!scalar) { - pQueryInfo->exprListLevelIndex += 1; - } - - *keepTableCols = false; - - size_t num = taosArrayGetSize(pParamList); - tExprNode** p = calloc(num, POINTER_BYTES); - - pQueryInfo->exprListLevelIndex += 1; - - for(int32_t i = 0; i < num; ++i) { - tSqlExprItem* pItem = taosArrayGet(pParamList, i); - int32_t code = sqlExprToExprNode(&p[i], pItem->pNode, pQueryInfo, pCols, keepTableCols, pMsgBuf); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - pQueryInfo->exprListLevelIndex -= 1; - int32_t outputIndex = (int32_t)getNumOfExprs(pQueryInfo); - - if (scalar) { - printf("scalar function found! %s\n", pSqlExpr->exprToken.z); - - // Expression on the results of aggregation functions - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TEXPR_FUNCTION_NODE; - - (*pExpr)->_function.pChild = p; - (*pExpr)->_function.num = num; - strncpy((*pExpr)->_function.functionName, pSqlExpr->Expr.operand.z, pSqlExpr->Expr.operand.n); - return TSDB_CODE_SUCCESS; - } else { - printf("agg function found, %s\n", pSqlExpr->exprToken.z); - tSqlExprItem item = {.pNode = (tSqlExpr*)pSqlExpr, .aliasName = NULL, .functionId = functionId}; - if (addAggExprAndResColumn(pQueryInfo, outputIndex, &item, false, pMsgBuf) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pQueryInfo->exprListLevelIndex -= 1; - // convert the aggregate function to be the input data columns for the outer function. - } + // convert the aggregate function to be the input data columns for the outer function. } - } + } if (pSqlExpr->pLeft == NULL) { // it is the leaf node assert(pSqlExpr->pRight == NULL); @@ -3089,23 +3205,7 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt return ret; } - pQueryInfo->curTableIdx = index.tableIndex; - STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - *pExpr = calloc(1, sizeof(tExprNode)); - (*pExpr)->nodeType = TEXPR_COL_NODE; - - (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); - *(*pExpr)->pSchema = *pSchema; - - if (*keepTableCols) { - SColumn c = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, index.type, (*pExpr)->pSchema); - taosArrayPush(pCols, &c); - } - - columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); + *pExpr = doCreateColumnNode(pQueryInfo, &index, *keepTableCols, pCols); return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { int32_t colType = -1; @@ -3141,7 +3241,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt } else { return buildInvalidOperationMsg(pMsgBuf, "not support filter expression"); } - } else { *pExpr = (tExprNode*)calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TEXPR_BINARYEXPR_NODE; @@ -3153,26 +3252,6 @@ int32_t sqlExprToExprNode(tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQuerySt (*pExpr)->_node.optr = convertRelationalOperator(&t); assert((*pExpr)->_node.optr != 0); - - // NOTE: binary|nchar data allows the >|< type filter - if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) { - if (pRight != NULL && pRight->nodeType == TEXPR_VALUE_NODE) { - if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL && pLeft->pSchema->type == TSDB_DATA_TYPE_BOOL) { - return buildInvalidOperationMsg(pMsgBuf, "invalid operator for bool"); - } - } - } - - // scalar op aggregate check - if (pLeft->nodeType == TEXPR_FUNCTION_NODE && pRight->nodeType != TEXPR_FUNCTION_NODE) { - return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); - } - - if (pLeft->nodeType == TEXPR_FUNCTION_NODE && pRight->nodeType == TEXPR_FUNCTION_NODE) { - if (qIsAggregateFunction(pLeft->_function.functionName) != qIsAggregateFunction(pRight->_function.functionName)) { - return buildInvalidOperationMsg(pMsgBuf, "invalid expression"); - } - } } return TSDB_CODE_SUCCESS; } @@ -3193,7 +3272,7 @@ static int32_t multiColumnListInsert(SQueryStmtInfo* pQueryInfo, SArray* pColumn columnListInsert(pQueryInfo->colList, p->uid, &s, p->flag); } - insertPrimaryTsColumn(pQueryInfo->colList, p1->uid); + insertPrimaryTsColumn(pQueryInfo->colList, NULL, p1->uid); return TSDB_CODE_SUCCESS; } @@ -3201,9 +3280,14 @@ static int32_t addScalarExprAndResColumn(SQueryStmtInfo* pQueryInfo, int32_t exp SArray* pColumnList = taosArrayInit(4, sizeof(SColumn)); SSchema s = createSchema(TSDB_DATA_TYPE_DOUBLE, sizeof(double), getNewResColId(), ""); + int32_t ret = validateSqlExpr(pItem->pNode, pQueryInfo, pMsgBuf); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + tExprNode* pNode = NULL; bool keepTableCols = true; - int32_t ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); + ret = sqlExprToExprNode(&pNode, pItem->pNode, pQueryInfo, pColumnList, &keepTableCols, pMsgBuf); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(pNode, NULL); return buildInvalidOperationMsg(pMsgBuf, "invalid expression in select clause"); diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 17dbde39da..e98443e2a1 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -815,13 +815,15 @@ SColumn* columnListInsert(SArray* pColumnList, uint64_t uid, SSchema* pSchema, i b->info.bytes = pSchema->bytes; b->info.type = pSchema->type; tstrncpy(b->name, pSchema->name, tListLen(b->name)); - taosArrayInsert(pColumnList, i, &b); + return b; } -SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid) { +SColumn* insertPrimaryTsColumn(SArray* pColumnList, const char* colName, uint64_t tableUid) { SSchema s = {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = TSDB_KEYSIZE, .colId = PRIMARYKEY_TIMESTAMP_COL_ID}; + strncpy(s.name, colName, tListLen(s.name)); + return columnListInsert(pColumnList, tableUid, &s, TSDB_COL_NORMAL); } diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 89e4166070..34d3639cc5 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -96,12 +96,13 @@ void generateLogicplan(const char* sql) { char* str = NULL; qQueryPlanToString(n, &str); + + printf("--------SQL:%s\n", sql); printf("%s\n", str); destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); destroySqlInfo(&info1); - } } @@ -164,10 +165,17 @@ TEST(testCase, planner_test) { } TEST(testCase, displayPlan) { - generateLogicplan("select count(*) from `t.1abc`"); - generateLogicplan("select count(*) from `t.1abc` group by a"); - generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); - generateLogicplan("select count(*),sum(a),avg(b),min(a+b) from `t.1abc`"); +// generateLogicplan("select count(*) from `t.1abc`"); +// generateLogicplan("select count(*)+ 22 from `t.1abc`"); +// generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h)"); +// generateLogicplan("select count(*) from `t.1abc` group by a"); +// generateLogicplan("select count(A+B) from `t.1abc` group by a"); +// generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); +// generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); +// generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); +// generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); +// generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); + generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc`"); // order by + group by column + limit offset + fill @@ -178,4 +186,9 @@ TEST(testCase, displayPlan) { // union + // Aggregate(count(*) [count(*) #5056], sum(a) [sum(a) #5057], avg(b) [avg(b) #5058], min(a+b) [min(a+b) #5060]) + // Projection(cols: [a+b #5059]) filters:(nil) + // Projection(cols: [ts #0], [a #1], [b #2]) filters:(nil) + // TableScan(t.1abc #110) time_range: -9223372036854775808 - 9223372036854775807 + } \ No newline at end of file diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 00693892e9..ffbae310f7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -46,12 +46,14 @@ typedef struct SJoinCond { static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); +static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo); int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { + exprInfoPushDown((struct SQueryStmtInfo*) pQueryInfo); SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); assert(taosArrayGetSize(upstream) == 1); @@ -134,12 +136,11 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla case QNODE_GROUPBY: { SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; - SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - pGroupbyExpr->tableIndex = p->tableIndex; - pGroupbyExpr->orderType = p->orderType; - pGroupbyExpr->orderIndex = p->orderIndex; + SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); + pGroupbyExpr->groupbyTag = p->groupbyTag; pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); + pNode->pExtInfo = pGroupbyExpr; break; } @@ -155,7 +156,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla break; } default: - assert(0); + break; } return pNode; @@ -188,12 +189,11 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); for (int32_t i = 0; i < numOfCols; ++i) { SColumn* pCol = taosArrayGetP(tableCols, i); - SSchema* pSchema = getOneColumnSchema(pTableMetaInfo1->pTableMeta, i); SSourceParam param = {0}; addIntoSourceParam(¶m, NULL, pCol); - - SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, pSchema, 0); + SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); + SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); pExpr[i] = p; } @@ -234,36 +234,47 @@ static SQueryPlanNode* createOneQueryPlanNode(SArray* p, SQueryPlanNode* pNode, } } -static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info, SArray** pExprs) { - // check for aggregation +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { + // group by column not by tag size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + // check for aggregation int32_t level = getFunctionLevel(pQueryInfo); + for(int32_t i = level - 1; i >= 0; --i) { SArray* p = pQueryInfo->exprList[i]; - SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); - if (i == 0) { + size_t num = taosArrayGetSize(p); + bool aggregateFunc = false; + for(int32_t j = 0; j < num; ++j) { + SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); + if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { + continue; + } + + aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); + if (aggregateFunc) { + break; + } + } + + if (aggregateFunc) { if (pQueryInfo->interval.interval > 0) { - int32_t numOfOutput = (int32_t)taosArrayGetSize(p); - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, numOfOutput, info, &pQueryInfo->interval); + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->interval); } else if (pQueryInfo->sessionWindow.gap > 0) { pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); } else if (pQueryInfo->stateWindow.columnId > 0) { pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, NULL, 0, info, NULL); + } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, info, &pQueryInfo->groupbyExpr); } else { - pNode = createOneQueryPlanNode(p, pNode, pExpr, info); + pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, info, NULL); } } else { - pNode = createOneQueryPlanNode(p, pNode, pExpr, info); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, info, NULL); } } - // group by column not by tag - if (numOfGroupCols != 0) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, NULL, 0, info, &pQueryInfo->groupbyExpr); - } - if (pQueryInfo->havingFieldNum > 0) { // int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); // pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL); @@ -285,7 +296,7 @@ static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryIn return pNode; } -static SQueryPlanNode* doCreateQueryPlanForOneTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, +static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, SArray* tableCols) { char name[TSDB_TABLE_FNAME_LEN] = {0}; tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); @@ -299,11 +310,62 @@ static SQueryPlanNode* doCreateQueryPlanForOneTable(SQueryStmtInfo* pQueryInfo, return pNode; } - SQueryPlanNode* pNode1 = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pExprs); + SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); tfree(info.tableName); return pNode1; } +static bool isAllAggExpr(SArray* pList) { + assert(pList != NULL); + + for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { + SExprInfo* p = taosArrayGetP(pList, k); + if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { + return false; + } + } + + return true; +} + +static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { + assert(pQueryInfo != NULL); + + size_t level = getFunctionLevel(pQueryInfo); + for(int32_t i = 0; i < level - 1; ++i) { + SArray* p = pQueryInfo->exprList[i]; + + SArray* pNext = pQueryInfo->exprList[i + 1]; + if (!isAllAggExpr(pNext)) { + continue; + } + + for (int32_t j = 0; j < taosArrayGetSize(p); ++j) { + SExprInfo* pExpr = taosArrayGetP(p, j); + + bool canPushDown = true; + if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE && qIsAggregateFunction(pExpr->pExpr->_function.functionName)) { + for (int32_t k = 0; k < taosArrayGetSize(pNext); ++k) { + SExprInfo* pNextLevelExpr = taosArrayGetP(pNext, k); + if (pExpr->base.pColumns->info.colId == pNextLevelExpr->base.resSchema.colId) { + // pExpr is dependent on the output of the under layer, so it can not be push downwards + canPushDown = false; + break; + } + } + } + + if (canPushDown) { + taosArrayInsert(pNext, j, &pExpr); + taosArrayRemove(p, j); + + // add the project function in level of "i" + + } + } + } +} + SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { SArray* upstream = NULL; @@ -357,12 +419,12 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { pQueryInfo->exprList[0]->pData, num, &info, NULL); // 4. add the aggregation or projection execution node - pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList); + pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); upstream = taosArrayInit(5, POINTER_BYTES); taosArrayPush(upstream, &pNode); } else { // only one table, normal query process STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryPlanNode* pNode = doCreateQueryPlanForOneTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); + SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); upstream = taosArrayInit(5, POINTER_BYTES); taosArrayPush(upstream, &pNode); } @@ -403,14 +465,30 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, switch(pQueryNode->info.type) { case QNODE_TABLESCAN: { STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 "\n", + len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 " cols: ", pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.uid, win->skey, win->ekey); + assert(len1 > 0); + len += len1; + + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); + len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); + + assert(len1 > 0); + len += len1; + } + + len1 = sprintf(buf + len, "\n"); + assert(len1 > 0); + len += len1; break; } case QNODE_PROJECT: { len1 = sprintf(buf + len, "cols: "); + assert(len1 > 0); + len += len1; for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { @@ -418,6 +496,8 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, SSqlExpr* p = &pExprInfo->base; len1 = sprintf(buf + len, "[%s #%d]", p->resSchema.name, p->resSchema.colId); + assert(len1 > 0); + len += len1; if (i < pQueryNode->numOfExpr - 1) { @@ -493,11 +573,16 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, } SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, 0); - - len1 = sprintf(buf + len,") groupby_col: [%s #%d]\n", pIndex->name, pIndex->colId); + len1 = sprintf(buf + len,") groupby_col: "); len += len1; + for(int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { + SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); + len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); + len += len1; + } + + len += sprintf(buf + len, "\n"); break; } diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index 1988fc9df7..61ad0a0498 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -189,7 +189,7 @@ static SQueryNode* doCreateQueryPlanForOneTableImpl(SQueryInfo* pQueryInfo, SQue return pNode; } -static SQueryNode* doCreateQueryPlanForOneTable(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, +static SQueryNode* doCreateQueryPlanForSingleTable(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, SArray* tableCols) { char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); @@ -266,7 +266,7 @@ SArray* createQueryPlanImpl(SQueryInfo* pQueryInfo) { taosArrayPush(upstream, &pNode); } else { // only one table, normal query process STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryNode* pNode = doCreateQueryPlanForOneTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList); + SQueryNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList); upstream = taosArrayInit(5, POINTER_BYTES); taosArrayPush(upstream, &pNode); } From 6fedb60ec276d143d4417aeac262a7fe5e8bd8dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Nov 2021 18:53:22 +0800 Subject: [PATCH 24/26] [td-10564] Add test case and fix bug in generated log query plan. --- include/common/common.h | 9 +++ include/common/taosmsg.h | 11 +--- source/libs/function/src/tfunction.c | 6 +- source/libs/parser/src/astValidate.c | 28 ++++----- source/libs/parser/test/plannerTest.cpp | 6 +- source/libs/planner/src/planner.c | 79 +++++++++++++++++++++---- 6 files changed, 96 insertions(+), 43 deletions(-) diff --git a/include/common/common.h b/include/common/common.h index be1a85e0e8..0913c12597 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -107,6 +107,15 @@ typedef struct SExprInfo { struct tExprNode *pExpr; } SExprInfo; +typedef struct SStateWindow { + SColumn col; +} SStateWindow; + +typedef struct SSessionWindow { + int64_t gap; // gap between two session window(in microseconds) + SColumn col; +} SSessionWindow; + #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2e0f59df04..4718d0e4b3 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -491,15 +491,6 @@ typedef struct SInterval { int64_t offset; } SInterval; -typedef struct SSessionWindow { - int64_t gap; // gap between two session window(in microseconds) - int32_t primaryColId; // primary timestamp column -} SSessionWindow; - -typedef struct SStateWindow { - int32_t columnId; -} SStateWindow; - typedef struct { SMsgHead head; char version[TSDB_VERSION_LEN]; @@ -524,7 +515,7 @@ typedef struct { int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; - SSessionWindow sw; // session window +// SSessionWindow sw; // session window uint16_t tagCondLen; // tag length in current query uint16_t colCondLen; // column length in current query int16_t numOfGroupCols; // num of group by columns diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 2e4a4a058e..d3fc19a47f 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -48,10 +48,10 @@ bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* funct bool qIsAggregateFunction(const char* functionName) { assert(functionName != NULL); - bool scalefunc = false; - qIsBuiltinFunction(functionName, strlen(functionName), &scalefunc); + bool scalarfunc = false; + qIsBuiltinFunction(functionName, strlen(functionName), &scalarfunc); - return !scalefunc; + return !scalarfunc; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 0aa7282e5c..46821a6e2b 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -763,6 +763,7 @@ int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSess const char* msg2 = "only one type time window allowed"; const char* msg3 = "invalid column name"; const char* msg4 = "invalid time window"; + const char* msg5 = "only the primary time stamp column can be used in session window"; // no session window if (!TPARSER_HAS_TOKEN(pSession->gap)) { @@ -795,18 +796,22 @@ int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSess } if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg5); } - pQueryInfo->sessionWindow.primaryColId = PRIMARYKEY_TIMESTAMP_COL_ID; + STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); + pQueryInfo->sessionWindow.col = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); return TSDB_CODE_SUCCESS; } // parse the window_state int32_t validateStateWindowNode(SQueryStmtInfo *pQueryInfo, SWindowStateVal* pWindowState, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid column name"; - const char* msg2 = "invalid column type"; - const char* msg3 = "not support state_window with group by "; + const char* msg2 = "invalid column type to create state window"; + const char* msg3 = "not support state_window with group by"; const char* msg4 = "function not support for super table query"; const char* msg5 = "not support state_window on tag column"; @@ -836,22 +841,15 @@ int32_t validateStateWindowNode(SQueryStmtInfo *pQueryInfo, SWindowStateVal* pWi return buildInvalidOperationMsg(pMsgBuf, msg5); } - if (pGroupExpr->columnInfo == NULL) { - pGroupExpr->columnInfo = taosArrayInit(4, sizeof(SColIndex)); - } - SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || IS_FLOAT_TYPE(pSchema->type)) { return buildInvalidOperationMsg(pMsgBuf, msg2); } - columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, TSDB_COL_NORMAL); - SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId }; - - //TODO use group by routine? state window query not support stable query. - taosArrayPush(pGroupExpr->columnInfo, &colIndex); + pQueryInfo->stateWindow.col = createColumn(pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); pQueryInfo->info.stateWindow = true; + columnListInsert(pQueryInfo->colList, pTableMeta->uid, pSchema, index.type); return TSDB_CODE_SUCCESS; } @@ -3047,10 +3045,10 @@ int32_t validateSqlExpr(const tSqlExpr* pSqlExpr, SQueryStmtInfo *pQueryInfo, SM if (pLeft->type == SQL_NODE_SQLFUNCTION && pRight->type == SQL_NODE_SQLFUNCTION) { char token[FUNCTIONS_NAME_MAX_LENGTH] = {0}; - tstrncpy(token, pLeft->Expr.operand.z, pLeft->Expr.operand.n); + strncpy(token, pLeft->Expr.operand.z, pLeft->Expr.operand.n); bool agg1 = qIsAggregateFunction(token); - tstrncpy(token, pRight->Expr.operand.z, pRight->Expr.operand.n); + strncpy(token, pRight->Expr.operand.z, pRight->Expr.operand.n); bool agg2 = qIsAggregateFunction(token); if (agg1 != agg2) { diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 34d3639cc5..ded7d73e85 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -167,7 +167,7 @@ TEST(testCase, planner_test) { TEST(testCase, displayPlan) { // generateLogicplan("select count(*) from `t.1abc`"); // generateLogicplan("select count(*)+ 22 from `t.1abc`"); -// generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h)"); +// generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30"); // generateLogicplan("select count(*) from `t.1abc` group by a"); // generateLogicplan("select count(A+B) from `t.1abc` group by a"); // generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); @@ -175,7 +175,9 @@ TEST(testCase, displayPlan) { // generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); // generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); // generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); - generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc`"); +// generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20"); +// generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)"); + generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)"); // order by + group by column + limit offset + fill diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index ffbae310f7..e70b4a1280 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -95,8 +95,7 @@ int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQuery //====================================================================================================================== static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo, - void* pExtInfo) { + SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo, void* pExtInfo) { SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); pNode->info.type = type; @@ -134,6 +133,20 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla break; } + case QNODE_STATEWINDOW: { + SColumn* psw = calloc(1, sizeof(SColumn)); + pNode->pExtInfo = psw; + memcpy(psw, pExtInfo, sizeof(SColumn)); + break; + } + + case QNODE_SESSIONWINDOW: { + SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); + pNode->pExtInfo = pSessionWindow; + memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); + break; + } + case QNODE_GROUPBY: { SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; @@ -262,9 +275,9 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer if (pQueryInfo->interval.interval > 0) { pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->interval); } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL); - } else if (pQueryInfo->stateWindow.columnId > 0) { - pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, NULL, 0, info, NULL); + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->sessionWindow); + } else if (pQueryInfo->stateWindow.col.info.colId > 0) { + pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, info, &pQueryInfo->stateWindow); } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, info, &pQueryInfo->groupbyExpr); } else { @@ -343,8 +356,8 @@ static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { for (int32_t j = 0; j < taosArrayGetSize(p); ++j) { SExprInfo* pExpr = taosArrayGetP(p, j); - bool canPushDown = true; if (pExpr->pExpr->nodeType == TEXPR_FUNCTION_NODE && qIsAggregateFunction(pExpr->pExpr->_function.functionName)) { + bool canPushDown = true; for (int32_t k = 0; k < taosArrayGetSize(pNext); ++k) { SExprInfo* pNextLevelExpr = taosArrayGetP(pNext, k); if (pExpr->base.pColumns->info.colId == pNextLevelExpr->base.resSchema.colId) { @@ -353,14 +366,14 @@ static void exprInfoPushDown(SQueryStmtInfo* pQueryInfo) { break; } } - } - if (canPushDown) { - taosArrayInsert(pNext, j, &pExpr); - taosArrayRemove(p, j); + if (canPushDown) { + taosArrayInsert(pNext, j, &pExpr); + taosArrayRemove(p, j); - // add the project function in level of "i" - + // todo add the project function in level of "i" + + } } } } @@ -390,7 +403,7 @@ SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { uint64_t uid = pTableMetaInfo->pTableMeta->uid; SArray* exprList = taosArrayInit(4, POINTER_BYTES); - if (copyExprInfoList(exprList, pQueryInfo->exprList, uid, true) != 0) { + if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // dropAllExprInfo(exprList); exit(-1); @@ -558,6 +571,46 @@ static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, break; } + case QNODE_STATEWINDOW: { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); + SSqlExpr* pExpr = &pExprInfo->base; + len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len,", "); + len += len1; + } + } + + len1 = sprintf(buf + len,") "); + len += len1; + + SColumn* pCol = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); + len += len1; + break; + } + + case QNODE_SESSIONWINDOW: { + for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); + SSqlExpr* pExpr = &pExprInfo->base; + len += sprintf(buf + len,"%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len,", "); + len += len1; + } + } + + len1 = sprintf(buf + len,") "); + len += len1; + + struct SSessionWindow* ps = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:[%s #%d], gap:%"PRId64" (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); + len += len1; + break; + } + case QNODE_GROUPBY: { // todo hide the invisible column for(int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); From 6886f9f727d34c5cc08c00ebd2f0f731bbf5eed7 Mon Sep 17 00:00:00 2001 From: Yiqing Liu Date: Wed, 10 Nov 2021 19:40:40 +0800 Subject: [PATCH 25/26] Revert "test for CI" --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a822c5b0e1..c19a720d49 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,8 +4,8 @@ import jenkins.model.CauseOfInterruption node { } -// def skipbuild=0 -// def win_stop=0 +def skipbuild=0 +def win_stop=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME From a0d3fa46c255dca3dcf1c2f522e068b94871e7b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 10 Nov 2021 23:20:01 +0800 Subject: [PATCH 26/26] [td-10564] refactor code and fix bug in parse sql. --- include/libs/function/function.h | 3 +- source/libs/parser/inc/astGenerator.h | 38 +- source/libs/parser/inc/sql.y | 6 +- source/libs/parser/src/astGenerator.c | 60 +- source/libs/parser/src/astValidate.c | 32 +- source/libs/parser/src/parser.c | 18 +- source/libs/parser/src/queryInfoUtil.c | 2 +- source/libs/parser/src/sql.c | 1215 +++++++++++---------- source/libs/parser/test/parserTests.cpp | 54 +- source/libs/parser/test/plannerTest.cpp | 32 +- source/libs/parser/test/tokenizerTest.cpp | 6 +- src/client/src/tscSQLParser.c | 30 +- src/query/inc/qSqlparser.h | 10 +- src/query/src/qSqlParser.c | 15 +- 14 files changed, 779 insertions(+), 742 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index f7d1fb9e45..92d8c972f5 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -235,11 +235,12 @@ typedef struct SMultiFunctionsDesc { bool hasFilter; bool onlyTagQuery; bool orderProjectQuery; - bool stateWindow; bool globalMerge; bool multigroupResult; bool blockDistribution; + bool stateWindow; bool timewindow; + bool sessionWindow; bool topbotQuery; bool interpQuery; bool distinct; diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 829112baf3..863c307f34 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -37,9 +37,14 @@ enum SQL_NODE_TYPE { SQL_NODE_EXPR = 4, }; -enum SQL_NODE_FROM_TYPE { - SQL_NODE_FROM_SUBQUERY = 1, - SQL_NODE_FROM_TABLELIST = 2, +enum SQL_FROM_NODE_TYPE { + SQL_FROM_NODE_SUBQUERY = 1, + SQL_FROM_NODE_TABLES = 2, +}; + +enum SQL_UNION_TYPE { + SQL_TYPE_UNIONALL = 1, + SQL_TYPE_UNION = 2, }; extern char tTokenTypeSwitcher[13]; @@ -79,8 +84,8 @@ typedef struct SWindowStateVal { struct SRelationInfo; typedef struct SSqlNode { - struct SArray *pSelNodeList; // select clause struct SRelationInfo *from; // from clause SArray + struct SArray *pSelNodeList; // select clause struct tSqlExpr *pWhere; // where clause [optional] SArray *pGroupby; // groupby clause, only for tags[optional], SArray SArray *pSortOrder; // orderby [optional], SArray @@ -95,18 +100,23 @@ typedef struct SSqlNode { struct tSqlExpr *pHaving; // having clause [optional] } SSqlNode; -typedef struct SRelElementPair { +typedef struct SSubclause { + int32_t unionType; + SArray *node; +} SSubclause; + +typedef struct SRelElement { union { - SToken tableName; - SArray *pSubquery; + SToken tableName; + SSubclause *pSubquery; }; SToken aliasName; -} SRelElementPair; +} SRelElement; typedef struct SRelationInfo { int32_t type; // nested query|table name list - SArray *list; // SArray + SArray *list; // SArray } SRelationInfo; typedef struct SCreatedTableInfo { @@ -216,7 +226,7 @@ typedef struct SMiscInfo { typedef struct SSqlInfo { int32_t type; bool valid; - SArray *list; // todo refactor + SSubclause sub; char msg[256]; SArray *funcs; union { @@ -257,7 +267,7 @@ SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrd SRelationInfo *setTableNameList(SRelationInfo *pRelationInfo, SToken *pName, SToken *pAlias); void * destroyRelationInfo(SRelationInfo *pFromInfo); -SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *pAlias); +SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SSubclause *pSub, SToken *pAlias); // sql expr leaf node tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType); @@ -285,13 +295,13 @@ SAlterTableInfo * tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray * SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, SToken *igExists); -void destroyAllSqlNode(SArray *pSqlNode); +void destroyAllSqlNode(struct SSubclause *pSqlNode); void destroySqlNode(SSqlNode *pSql); void freeCreateTableInfo(void* p); SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type); -SArray *setSubclause(SArray *pList, void *pSqlNode); -SArray *appendSelectClause(SArray *pList, void *pSubclause); +SSubclause* setSubclause(SSubclause* sub, void *pSqlNode); +SSubclause* appendSelectClause(SSubclause *sub, int32_t unionType, void *pSubclause); void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists); void* destroyCreateTableSql(SCreateTableSql* pCreate); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index a7577c4e4b..04fb3beb54 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -487,11 +487,11 @@ select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_option(K) sl select(A) ::= LP select(B) RP. {A = B;} -%type union {SArray*} +%type union {SSubclause*} %destructor union {destroyAllSqlNode($$);} union(Y) ::= select(X). { Y = setSubclause(NULL, X); } -union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } - +union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, SQL_TYPE_UNIONALL, X); } +union(Y) ::= union(Z) UNION select(X). { Y = appendSelectClause(Z, SQL_TYPE_UNION, X); } cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } // Support for the SQL exprssion without from & where subclauses, e.g., diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index c574f75d07..53d05c87b3 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -72,11 +72,11 @@ SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrd SRelationInfo *setTableNameList(SRelationInfo *pRelationInfo, SToken *pName, SToken *pAlias) { if (pRelationInfo == NULL) { pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); } - pRelationInfo->type = SQL_NODE_FROM_TABLELIST; - SRelElementPair p = {.tableName = *pName}; + pRelationInfo->type = SQL_FROM_NODE_TABLES; + SRelElement p = {.tableName = *pName}; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -92,12 +92,12 @@ void *destroyRelationInfo(SRelationInfo *pRelationInfo) { return NULL; } - if (pRelationInfo->type == SQL_NODE_FROM_TABLELIST) { + if (pRelationInfo->type == SQL_FROM_NODE_TABLES) { taosArrayDestroy(pRelationInfo->list); } else { size_t size = taosArrayGetSize(pRelationInfo->list); for(int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pRelationInfo->list, i); + SSubclause* pa = taosArrayGetP(pRelationInfo->list, i); destroyAllSqlNode(pa); } taosArrayDestroy(pRelationInfo->list); @@ -107,15 +107,15 @@ void *destroyRelationInfo(SRelationInfo *pRelationInfo) { return NULL; } -SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *pAlias) { +SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SSubclause *pSub, SToken *pAlias) { if (pRelationInfo == NULL) { pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); } - pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; + pRelationInfo->type = SQL_FROM_NODE_SUBQUERY; - SRelElementPair p = {.pSubquery = pSub}; + SRelElement p = {.pSubquery = pSub}; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -641,18 +641,18 @@ SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, return info; } -void destroyAllSqlNode(SArray *pList) { - if (pList == NULL) { +void destroyAllSqlNode(struct SSubclause *pSub) { + if (pSub->node == NULL) { return; } - size_t size = taosArrayGetSize(pList); + size_t size = taosArrayGetSize(pSub->node); for(int32_t i = 0; i < size; ++i) { - SSqlNode *pNode = taosArrayGetP(pList, i); + SSqlNode *pNode = taosArrayGetP(pSub->node, i); destroySqlNode(pNode); } - taosArrayDestroy(pList); + taosArrayDestroy(pSub->node); } static void freeItem(void *pItem) { @@ -698,7 +698,8 @@ SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, in pInfo->type = type; if (type == TSDB_SQL_SELECT) { - pInfo->list = (SArray*) pSqlExprInfo; + pInfo->sub = *(SSubclause*) pSqlExprInfo; + tfree(pSqlExprInfo); } else { pInfo->pCreateTableInfo = pSqlExprInfo; } @@ -710,18 +711,25 @@ SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, in return pInfo; } -SArray* setSubclause(SArray* pList, void *pSqlNode) { - if (pList == NULL) { - pList = taosArrayInit(1, POINTER_BYTES); +SSubclause* setSubclause(SSubclause* pSub, void *pSqlNode) { + if (pSub == NULL) { + pSub = malloc(sizeof(SSubclause)); + + pSub->unionType = SQL_TYPE_UNIONALL; + pSub->node = taosArrayInit(1, POINTER_BYTES); } - taosArrayPush(pList, &pSqlNode); - return pList; + taosArrayPush(pSub->node, &pSqlNode); + return pSub; } -SArray* appendSelectClause(SArray *pList, void *pSubclause) { - taosArrayPush(pList, &pSubclause); - return pList; +SSubclause* appendSelectClause(SSubclause *pSub, int32_t type, void *pSubclause) { + taosArrayPush(pSub->node, &pSubclause); + if (type == SQL_TYPE_UNION) { + pSub->unionType = type; + } + + return pSub; } void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists) { @@ -776,7 +784,7 @@ void destroySqlInfo(SSqlInfo *pInfo) { taosArrayDestroy(pInfo->funcs); if (pInfo->type == TSDB_SQL_SELECT) { - destroyAllSqlNode(pInfo->list); + destroyAllSqlNode(&pInfo->sub); } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { @@ -785,7 +793,7 @@ void destroySqlInfo(SSqlInfo *pInfo) { tfree(pInfo->pAlterInfo->tagData.data); tfree(pInfo->pAlterInfo); } else if (pInfo->type == TSDB_SQL_COMPACT_VNODE) { - tSqlExprListDestroy(pInfo->list); + tSqlExprListDestroy(pInfo->sub.node); } else { if (pInfo->pMiscInfo != NULL) { taosArrayDestroy(pInfo->pMiscInfo->a); @@ -935,7 +943,7 @@ void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SToken *pName, SToken* pPwd, void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) { pInfo->type = type; - pInfo->list = pParam; + pInfo->sub.node = pParam; } void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 46821a6e2b..8e0f59b07e 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -268,11 +268,11 @@ void destroyQueryInfo(SQueryStmtInfo* pQueryInfo) { } static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { - SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, index); + SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, index); // union all is not support currently - SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - if (taosArrayGetSize(subInfo->pSubquery) >= 2) { + SSqlNode* p = taosArrayGetP(subInfo->pSubquery->node, 0); + if (taosArrayGetSize(subInfo->pSubquery->node) >= 2) { return buildInvalidOperationMsg(pMsgBuf, "not support union in subquery"); } @@ -804,6 +804,7 @@ int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSess SSchema* pSchema = getOneColumnSchema(pTableMeta, index.columnIndex); pQueryInfo->sessionWindow.col = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, pSchema); + pQueryInfo->info.sessionWindow = true; return TSDB_CODE_SUCCESS; } @@ -1401,13 +1402,13 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* // return doLocalQueryProcess(pCmd, pQueryInfo, pSqlNode); } - if (pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + if (pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { pQueryInfo->numOfTables = 0; // parse the subquery in the first place int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); for (int32_t i = 0; i < numOfSub; ++i) { - SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, i); + SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, i); code = doValidateSubquery(pSqlNode, i, pQueryInfo, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1574,7 +1575,8 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { const char* msg5 = "scalar function can not be used in time window query"; const char* msg6 = "not support distinct mixed with join"; const char* msg7 = "not support distinct mixed with groupby"; - const char* msg8 = "_block_dist not support subquery, only support stable/table"; + const char* msg8 = "block_dist not support subquery, only support stable/table"; + const char* msg9 = "time window aggregate can not be mixed up with group by column"; if (pQueryInfo->info.topbotQuery) { @@ -1656,6 +1658,15 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { * nested subquery not support block_dist query * select block_dist() from (select * from table_name) */ + + /* + * 8. invalid sql: + * select count(*) from table_name [interval(10s)|session(ts, 10s)|state_window(col_name)] group by col_name + */ + if ((pQueryInfo->info.timewindow || pQueryInfo->info.stateWindow || pQueryInfo->info.sessionWindow) && + pQueryInfo->info.groupbyColumn) { + return buildInvalidOperationMsg(pMsgBuf, msg9); + } } static int32_t resColId = 5000; @@ -3790,7 +3801,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); - if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { return setInvalidOperatorMsg(pMsgBuf, msg1); } @@ -3895,9 +3906,9 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SMsgBuf buf = {.buf = msgBuf, .len = msgBufLen}; - size_t len = taosArrayGetSize(pInfo->list); + size_t len = taosArrayGetSize(pInfo->sub.node); for(int32_t i = 0; i < len; ++i) { - SSqlNode* p = taosArrayGetP(pInfo->list, i); + SSqlNode* p = taosArrayGetP(pInfo->sub.node, i); code = evaluateSqlNode(p, pTableMeta->tableInfo.precision, &buf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3905,7 +3916,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } for(int32_t i = 0; i < len; ++i) { - SSqlNode* p = taosArrayGetP(pInfo->list, i); + SSqlNode* p = taosArrayGetP(pInfo->sub.node, i); validateSqlNode(p, pQueryInfo, &buf); } @@ -3916,6 +3927,5 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer return code; } - // convert the sqlnode into queryinfo return code; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 0585dc8905..29561f7f54 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -82,12 +82,12 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); for (int32_t j = 0; j < numOfSub; ++j) { - SRelElementPair* sub = taosArrayGet(pSqlNode->from->list, j); + SRelElement* sub = taosArrayGet(pSqlNode->from->list, j); - int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery); + int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery->node); for (int32_t i = 0; i < num; ++i) { - SSqlNode* p = taosArrayGetP(sub->pSubquery, i); - if (p->from->type == SQL_NODE_FROM_TABLELIST) { + SSqlNode* p = taosArrayGetP(sub->pSubquery->node, i); + if (p->from->type == SQL_FROM_NODE_TABLES) { int32_t code = getTableNameFromSqlNode(p, tableNameList, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; @@ -105,10 +105,10 @@ int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SMsgB const char* msg1 = "invalid table name"; int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); - assert(pSqlNode->from->type == SQL_NODE_FROM_TABLELIST); + assert(pSqlNode->from->type == SQL_FROM_NODE_TABLES); for(int32_t j = 0; j < numOfTables; ++j) { - SRelElementPair* item = taosArrayGet(pSqlNode->from->list, j); + SRelElement* item = taosArrayGet(pSqlNode->from->list, j); SToken* t = &item->tableName; if (t->type == TK_INTEGER || t->type == TK_FLOAT || t->type == TK_STRING) { @@ -138,15 +138,15 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet pMetaInfo->pTableName = taosArrayInit(4, sizeof(SName)); pMetaInfo->pUdf = taosArrayInit(4, POINTER_BYTES); - size_t size = taosArrayGetSize(pSqlInfo->list); + size_t size = taosArrayGetSize(pSqlInfo->sub.node); for (int32_t i = 0; i < size; ++i) { - SSqlNode* pSqlNode = taosArrayGetP(pSqlInfo->list, i); + SSqlNode* pSqlNode = taosArrayGetP(pSqlInfo->sub.node, i); if (pSqlNode->from == NULL) { return buildInvalidOperationMsg(&msgBuf, "invalid from clause"); } // load the table meta in the FROM clause - if (pSqlNode->from->type == SQL_NODE_FROM_TABLELIST) { + if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { code = getTableNameFromSqlNode(pSqlNode, pMetaInfo->pTableName, &msgBuf); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 91017ec631..04922ce5ad 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -182,7 +182,7 @@ void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index) { assert(pQueryInfo != NULL && pQueryInfo->exprList && index >= 0); - return taosArrayGetP(getCurrentExprList(pQueryInfo->exprList), index); + return taosArrayGetP(getCurrentExprList(pQueryInfo), index); } void destroyExprInfo(SExprInfo* pExprInfo) { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 2b12be2a17..7158708989 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -34,7 +34,6 @@ #include "tmsgtype.h" #include "ttoken.h" #include "ttokendef.h" -//#include "tutil.h" #include "tvariant.h" /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols @@ -120,6 +119,7 @@ typedef union { SSessionWindowVal yy511; tSqlExpr* yy526; int64_t yy531; + SSubclause* yy551; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -130,16 +130,16 @@ typedef union { #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 #define YYNSTATE 368 -#define YYNRULE 294 +#define YYNRULE 295 #define YYNTOKEN 197 #define YY_MAX_SHIFT 367 -#define YY_MIN_SHIFTREDUCE 576 -#define YY_MAX_SHIFTREDUCE 869 -#define YY_ERROR_ACTION 870 -#define YY_ACCEPT_ACTION 871 -#define YY_NO_ACTION 872 -#define YY_MIN_REDUCE 873 -#define YY_MAX_REDUCE 1166 +#define YY_MIN_SHIFTREDUCE 577 +#define YY_MAX_SHIFTREDUCE 871 +#define YY_ERROR_ACTION 872 +#define YY_ACCEPT_ACTION 873 +#define YY_NO_ACTION 874 +#define YY_MIN_REDUCE 875 +#define YY_MAX_REDUCE 1169 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -205,86 +205,86 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (773) +#define YY_ACTTAB_COUNT (776) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 23, 628, 366, 235, 1051, 208, 241, 712, 211, 629, - /* 10 */ 1029, 871, 367, 59, 60, 173, 63, 64, 1042, 1142, - /* 20 */ 255, 53, 52, 51, 628, 62, 324, 67, 65, 68, - /* 30 */ 66, 157, 629, 286, 238, 58, 57, 344, 343, 56, - /* 40 */ 55, 54, 59, 60, 247, 63, 64, 252, 1029, 255, - /* 50 */ 53, 52, 51, 664, 62, 324, 67, 65, 68, 66, - /* 60 */ 999, 1042, 997, 998, 58, 57, 752, 1000, 56, 55, - /* 70 */ 54, 1001, 1048, 1002, 1003, 58, 57, 277, 1015, 56, - /* 80 */ 55, 54, 59, 60, 164, 63, 64, 38, 82, 255, - /* 90 */ 53, 52, 51, 88, 62, 324, 67, 65, 68, 66, - /* 100 */ 284, 283, 249, 322, 58, 57, 1029, 211, 56, 55, - /* 110 */ 54, 38, 59, 61, 806, 63, 64, 1042, 1143, 255, - /* 120 */ 53, 52, 51, 628, 62, 324, 67, 65, 68, 66, - /* 130 */ 45, 629, 237, 239, 58, 57, 1026, 164, 56, 55, - /* 140 */ 54, 60, 1023, 63, 64, 771, 772, 255, 53, 52, - /* 150 */ 51, 756, 62, 324, 67, 65, 68, 66, 164, 1090, - /* 160 */ 1025, 296, 58, 57, 322, 100, 56, 55, 54, 577, - /* 170 */ 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, - /* 180 */ 588, 589, 590, 155, 354, 236, 63, 64, 209, 248, - /* 190 */ 255, 53, 52, 51, 628, 62, 324, 67, 65, 68, - /* 200 */ 66, 1017, 629, 164, 1028, 58, 57, 362, 960, 56, - /* 210 */ 55, 54, 1089, 44, 320, 361, 360, 319, 318, 317, - /* 220 */ 359, 316, 315, 314, 358, 313, 357, 356, 154, 152, - /* 230 */ 151, 298, 24, 93, 991, 979, 980, 981, 982, 983, - /* 240 */ 984, 985, 986, 987, 988, 989, 990, 992, 993, 214, - /* 250 */ 749, 254, 821, 922, 125, 810, 222, 813, 215, 816, - /* 260 */ 192, 97, 139, 138, 137, 221, 354, 254, 821, 329, - /* 270 */ 88, 810, 251, 813, 269, 816, 9, 29, 94, 67, - /* 280 */ 65, 68, 66, 273, 272, 233, 234, 58, 57, 325, - /* 290 */ 81, 56, 55, 54, 1012, 1013, 35, 1016, 812, 216, - /* 300 */ 815, 233, 234, 258, 5, 41, 182, 45, 56, 55, - /* 310 */ 54, 181, 106, 111, 102, 110, 38, 263, 736, 38, - /* 320 */ 932, 733, 217, 734, 1014, 735, 95, 192, 256, 276, - /* 330 */ 309, 80, 211, 38, 69, 123, 117, 128, 229, 811, - /* 340 */ 83, 814, 127, 1143, 133, 136, 126, 202, 200, 198, - /* 350 */ 69, 260, 261, 130, 197, 143, 142, 141, 140, 38, - /* 360 */ 44, 245, 361, 360, 246, 1026, 788, 359, 1026, 822, - /* 370 */ 817, 358, 38, 357, 356, 38, 818, 38, 333, 259, - /* 380 */ 38, 257, 1026, 332, 331, 822, 817, 1137, 211, 365, - /* 390 */ 364, 148, 818, 265, 38, 262, 38, 339, 338, 1143, - /* 400 */ 264, 264, 264, 14, 334, 85, 76, 96, 1026, 923, - /* 410 */ 86, 178, 179, 1027, 3, 193, 192, 335, 278, 819, - /* 420 */ 336, 1026, 340, 787, 1026, 341, 1026, 1, 180, 1026, - /* 430 */ 768, 73, 39, 326, 778, 737, 738, 99, 779, 342, - /* 440 */ 722, 346, 280, 1026, 159, 1026, 77, 280, 70, 301, - /* 450 */ 724, 303, 26, 723, 39, 34, 808, 844, 823, 253, - /* 460 */ 820, 627, 1136, 39, 70, 98, 79, 70, 16, 1100, - /* 470 */ 15, 25, 25, 74, 116, 25, 115, 18, 741, 17, - /* 480 */ 742, 6, 739, 20, 740, 19, 1135, 122, 304, 121, - /* 490 */ 22, 231, 21, 232, 809, 135, 134, 212, 711, 213, - /* 500 */ 274, 218, 210, 219, 220, 224, 225, 1162, 226, 223, - /* 510 */ 207, 1154, 1099, 243, 1096, 1095, 244, 345, 156, 48, - /* 520 */ 1082, 1081, 174, 1050, 1061, 153, 1058, 1043, 1059, 281, - /* 530 */ 1024, 1063, 310, 285, 240, 158, 287, 825, 163, 292, - /* 540 */ 165, 175, 1022, 1040, 176, 167, 177, 937, 306, 767, - /* 550 */ 307, 308, 311, 166, 312, 289, 46, 205, 42, 323, - /* 560 */ 931, 330, 1161, 113, 78, 75, 299, 1160, 50, 1157, - /* 570 */ 183, 297, 168, 295, 337, 1153, 293, 119, 1152, 1149, - /* 580 */ 184, 957, 291, 43, 40, 47, 206, 919, 129, 917, - /* 590 */ 131, 132, 915, 914, 266, 195, 196, 288, 911, 910, - /* 600 */ 909, 908, 907, 906, 905, 199, 201, 902, 900, 898, - /* 610 */ 896, 203, 893, 204, 889, 355, 49, 124, 279, 84, - /* 620 */ 347, 89, 290, 1083, 348, 349, 350, 351, 352, 353, - /* 630 */ 230, 363, 250, 869, 267, 268, 305, 868, 270, 271, - /* 640 */ 867, 850, 227, 849, 280, 275, 228, 107, 936, 935, - /* 650 */ 10, 108, 300, 744, 282, 87, 30, 90, 769, 913, - /* 660 */ 912, 160, 904, 187, 191, 144, 958, 185, 186, 188, - /* 670 */ 189, 145, 146, 190, 903, 2, 780, 995, 147, 161, - /* 680 */ 169, 170, 959, 171, 172, 33, 895, 894, 774, 162, - /* 690 */ 4, 1005, 91, 242, 776, 92, 294, 31, 11, 32, - /* 700 */ 13, 12, 27, 302, 28, 99, 101, 642, 36, 103, - /* 710 */ 104, 37, 105, 677, 675, 674, 673, 671, 670, 669, - /* 720 */ 666, 321, 109, 632, 7, 826, 824, 8, 328, 327, - /* 730 */ 112, 114, 71, 72, 118, 714, 39, 120, 713, 710, - /* 740 */ 658, 656, 648, 654, 650, 652, 646, 644, 680, 679, - /* 750 */ 678, 676, 672, 668, 667, 194, 630, 594, 873, 872, - /* 760 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, - /* 770 */ 872, 149, 150, + /* 0 */ 23, 629, 366, 236, 1054, 209, 242, 713, 212, 630, + /* 10 */ 1031, 873, 367, 59, 60, 174, 63, 64, 1044, 1145, + /* 20 */ 256, 53, 52, 51, 629, 62, 324, 67, 65, 68, + /* 30 */ 66, 158, 630, 286, 239, 58, 57, 344, 343, 56, + /* 40 */ 55, 54, 59, 60, 248, 63, 64, 253, 1031, 256, + /* 50 */ 53, 52, 51, 665, 62, 324, 67, 65, 68, 66, + /* 60 */ 1001, 1044, 999, 1000, 58, 57, 210, 1002, 56, 55, + /* 70 */ 54, 1003, 1051, 1004, 1005, 58, 57, 278, 216, 56, + /* 80 */ 55, 54, 59, 60, 165, 63, 64, 38, 83, 256, + /* 90 */ 53, 52, 51, 89, 62, 324, 67, 65, 68, 66, + /* 100 */ 284, 283, 250, 754, 58, 57, 1031, 212, 56, 55, + /* 110 */ 54, 38, 59, 61, 808, 63, 64, 1044, 1146, 256, + /* 120 */ 53, 52, 51, 629, 62, 324, 67, 65, 68, 66, + /* 130 */ 45, 630, 238, 240, 58, 57, 1028, 165, 56, 55, + /* 140 */ 54, 60, 1025, 63, 64, 773, 774, 256, 53, 52, + /* 150 */ 51, 96, 62, 324, 67, 65, 68, 66, 165, 1093, + /* 160 */ 1027, 296, 58, 57, 89, 84, 56, 55, 54, 578, + /* 170 */ 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, + /* 180 */ 589, 590, 591, 156, 322, 237, 63, 64, 758, 249, + /* 190 */ 256, 53, 52, 51, 629, 62, 324, 67, 65, 68, + /* 200 */ 66, 45, 630, 88, 1017, 58, 57, 362, 962, 56, + /* 210 */ 55, 54, 1092, 44, 320, 361, 360, 319, 318, 317, + /* 220 */ 359, 316, 315, 314, 358, 313, 357, 356, 155, 153, + /* 230 */ 152, 298, 24, 94, 993, 981, 982, 983, 984, 985, + /* 240 */ 986, 987, 988, 989, 990, 991, 992, 994, 995, 215, + /* 250 */ 751, 255, 823, 924, 126, 812, 223, 815, 354, 818, + /* 260 */ 193, 98, 140, 139, 138, 222, 354, 255, 823, 329, + /* 270 */ 89, 812, 252, 815, 270, 818, 9, 29, 217, 67, + /* 280 */ 65, 68, 66, 274, 273, 234, 235, 58, 57, 325, + /* 290 */ 322, 56, 55, 54, 1014, 1015, 35, 1018, 814, 218, + /* 300 */ 817, 234, 235, 259, 5, 41, 183, 45, 56, 55, + /* 310 */ 54, 182, 107, 112, 103, 111, 38, 264, 737, 38, + /* 320 */ 934, 734, 742, 735, 743, 736, 165, 193, 257, 277, + /* 330 */ 309, 81, 212, 38, 69, 124, 118, 129, 230, 813, + /* 340 */ 1140, 816, 128, 1146, 134, 137, 127, 203, 201, 199, + /* 350 */ 69, 261, 262, 131, 198, 144, 143, 142, 141, 38, + /* 360 */ 44, 246, 361, 360, 247, 1028, 101, 359, 1028, 824, + /* 370 */ 819, 358, 38, 357, 356, 38, 820, 38, 333, 260, + /* 380 */ 38, 258, 1028, 332, 331, 824, 819, 790, 212, 365, + /* 390 */ 364, 149, 820, 266, 38, 263, 38, 339, 338, 1146, + /* 400 */ 265, 95, 1019, 14, 334, 265, 82, 97, 1028, 86, + /* 410 */ 77, 179, 265, 3, 194, 87, 180, 335, 326, 821, + /* 420 */ 336, 1028, 340, 1029, 1028, 341, 1028, 925, 279, 1028, + /* 430 */ 74, 1, 181, 770, 193, 738, 739, 100, 34, 342, + /* 440 */ 1016, 346, 39, 1028, 789, 1028, 73, 160, 780, 781, + /* 450 */ 78, 723, 73, 301, 725, 303, 724, 810, 254, 846, + /* 460 */ 822, 825, 70, 26, 628, 39, 80, 39, 70, 99, + /* 470 */ 70, 304, 75, 25, 16, 25, 15, 1139, 25, 117, + /* 480 */ 6, 116, 18, 1138, 17, 740, 20, 741, 19, 123, + /* 490 */ 232, 122, 22, 233, 21, 811, 136, 135, 712, 213, + /* 500 */ 275, 214, 219, 211, 220, 221, 225, 1030, 226, 227, + /* 510 */ 224, 208, 1165, 1157, 1103, 1046, 1102, 244, 1099, 1098, + /* 520 */ 245, 345, 827, 157, 48, 1053, 1064, 1085, 1061, 1062, + /* 530 */ 1045, 1084, 281, 1066, 1026, 154, 159, 164, 292, 175, + /* 540 */ 176, 1024, 33, 285, 177, 168, 178, 939, 769, 306, + /* 550 */ 307, 308, 311, 312, 1042, 46, 206, 42, 323, 933, + /* 560 */ 330, 1164, 114, 241, 79, 1163, 287, 1160, 289, 76, + /* 570 */ 166, 299, 184, 337, 167, 1156, 50, 120, 1155, 297, + /* 580 */ 1152, 185, 295, 959, 43, 40, 47, 207, 921, 130, + /* 590 */ 919, 132, 133, 917, 916, 267, 293, 196, 197, 913, + /* 600 */ 912, 911, 910, 291, 909, 908, 907, 200, 202, 904, + /* 610 */ 902, 900, 898, 204, 895, 205, 288, 891, 49, 310, + /* 620 */ 280, 85, 90, 290, 355, 1086, 348, 125, 347, 349, + /* 630 */ 350, 351, 231, 352, 251, 305, 353, 363, 871, 269, + /* 640 */ 870, 268, 271, 228, 229, 108, 938, 937, 109, 272, + /* 650 */ 869, 852, 276, 851, 73, 10, 915, 300, 282, 914, + /* 660 */ 745, 145, 188, 146, 187, 960, 186, 189, 190, 192, + /* 670 */ 191, 906, 905, 961, 147, 997, 2, 30, 148, 897, + /* 680 */ 896, 91, 171, 169, 172, 170, 173, 1007, 771, 4, + /* 690 */ 161, 163, 782, 162, 243, 776, 92, 31, 778, 93, + /* 700 */ 294, 11, 12, 32, 13, 27, 302, 28, 100, 102, + /* 710 */ 105, 36, 104, 643, 37, 106, 678, 676, 675, 674, + /* 720 */ 672, 671, 670, 667, 633, 321, 110, 7, 327, 328, + /* 730 */ 828, 39, 826, 8, 113, 71, 115, 72, 715, 714, + /* 740 */ 119, 711, 659, 121, 657, 649, 655, 651, 653, 647, + /* 750 */ 645, 681, 680, 679, 677, 673, 669, 668, 195, 631, + /* 760 */ 595, 875, 874, 874, 874, 874, 874, 874, 874, 874, + /* 770 */ 874, 874, 874, 874, 150, 151, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 267, 1, 200, 201, 200, 267, 246, 5, 267, 9, @@ -293,78 +293,78 @@ static const YYCODETYPE yy_lookahead[] = { /* 30 */ 30, 200, 9, 272, 264, 35, 36, 35, 36, 39, /* 40 */ 40, 41, 13, 14, 246, 16, 17, 207, 250, 20, /* 50 */ 21, 22, 23, 5, 25, 26, 27, 28, 29, 30, - /* 60 */ 224, 248, 226, 227, 35, 36, 39, 231, 39, 40, - /* 70 */ 41, 235, 268, 237, 238, 35, 36, 264, 0, 39, + /* 60 */ 224, 248, 226, 227, 35, 36, 267, 231, 39, 40, + /* 70 */ 41, 235, 268, 237, 238, 35, 36, 264, 267, 39, /* 80 */ 40, 41, 13, 14, 200, 16, 17, 200, 88, 20, /* 90 */ 21, 22, 23, 84, 25, 26, 27, 28, 29, 30, - /* 100 */ 269, 270, 246, 86, 35, 36, 250, 267, 39, 40, + /* 100 */ 269, 270, 246, 39, 35, 36, 250, 267, 39, 40, /* 110 */ 41, 200, 13, 14, 85, 16, 17, 248, 278, 20, /* 120 */ 21, 22, 23, 1, 25, 26, 27, 28, 29, 30, /* 130 */ 121, 9, 245, 264, 35, 36, 249, 200, 39, 40, /* 140 */ 41, 14, 200, 16, 17, 127, 128, 20, 21, 22, - /* 150 */ 23, 124, 25, 26, 27, 28, 29, 30, 200, 275, - /* 160 */ 249, 277, 35, 36, 86, 208, 39, 40, 41, 47, + /* 150 */ 23, 251, 25, 26, 27, 28, 29, 30, 200, 275, + /* 160 */ 249, 277, 35, 36, 84, 265, 39, 40, 41, 47, /* 170 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 180 */ 58, 59, 60, 61, 92, 63, 16, 17, 267, 247, + /* 180 */ 58, 59, 60, 61, 86, 63, 16, 17, 124, 247, /* 190 */ 20, 21, 22, 23, 1, 25, 26, 27, 28, 29, - /* 200 */ 30, 244, 9, 200, 250, 35, 36, 222, 223, 39, + /* 200 */ 30, 121, 9, 123, 0, 35, 36, 222, 223, 39, /* 210 */ 40, 41, 275, 100, 101, 102, 103, 104, 105, 106, /* 220 */ 107, 108, 109, 110, 111, 112, 113, 114, 64, 65, /* 230 */ 66, 273, 46, 275, 224, 225, 226, 227, 228, 229, /* 240 */ 230, 231, 232, 233, 234, 235, 236, 237, 238, 63, - /* 250 */ 99, 1, 2, 206, 80, 5, 70, 7, 267, 9, + /* 250 */ 99, 1, 2, 206, 80, 5, 70, 7, 92, 9, /* 260 */ 213, 208, 76, 77, 78, 79, 92, 1, 2, 83, - /* 270 */ 84, 5, 207, 7, 144, 9, 125, 84, 275, 27, + /* 270 */ 84, 5, 207, 7, 144, 9, 125, 84, 267, 27, /* 280 */ 28, 29, 30, 153, 154, 35, 36, 35, 36, 39, - /* 290 */ 208, 39, 40, 41, 241, 242, 243, 244, 5, 267, + /* 290 */ 86, 39, 40, 41, 241, 242, 243, 244, 5, 267, /* 300 */ 7, 35, 36, 70, 64, 65, 66, 121, 39, 40, /* 310 */ 41, 71, 72, 73, 74, 75, 200, 70, 2, 200, - /* 320 */ 206, 5, 267, 7, 242, 9, 251, 213, 207, 143, + /* 320 */ 206, 5, 5, 7, 7, 9, 200, 213, 207, 143, /* 330 */ 90, 145, 267, 200, 84, 64, 65, 66, 152, 5, - /* 340 */ 265, 7, 71, 278, 73, 74, 75, 64, 65, 66, + /* 340 */ 267, 7, 71, 278, 73, 74, 75, 64, 65, 66, /* 350 */ 84, 35, 36, 82, 71, 72, 73, 74, 75, 200, - /* 360 */ 100, 245, 102, 103, 245, 249, 78, 107, 249, 119, + /* 360 */ 100, 245, 102, 103, 245, 249, 208, 107, 249, 119, /* 370 */ 120, 111, 200, 113, 114, 200, 126, 200, 245, 146, - /* 380 */ 200, 148, 249, 150, 151, 119, 120, 267, 267, 67, + /* 380 */ 200, 148, 249, 150, 151, 119, 120, 78, 267, 67, /* 390 */ 68, 69, 126, 146, 200, 148, 200, 150, 151, 278, - /* 400 */ 200, 200, 200, 84, 245, 85, 99, 88, 249, 206, - /* 410 */ 85, 211, 211, 211, 204, 205, 213, 245, 85, 126, - /* 420 */ 245, 249, 245, 135, 249, 245, 249, 209, 210, 249, - /* 430 */ 85, 99, 99, 15, 85, 119, 120, 118, 85, 245, - /* 440 */ 85, 245, 122, 249, 99, 249, 139, 122, 99, 85, - /* 450 */ 85, 85, 99, 85, 99, 84, 1, 85, 85, 62, - /* 460 */ 126, 85, 267, 99, 99, 99, 84, 99, 147, 240, - /* 470 */ 149, 99, 99, 141, 147, 99, 149, 147, 5, 149, - /* 480 */ 7, 84, 5, 147, 7, 149, 267, 147, 117, 149, - /* 490 */ 147, 267, 149, 267, 39, 80, 81, 267, 116, 267, + /* 400 */ 200, 275, 244, 84, 245, 200, 208, 88, 249, 85, + /* 410 */ 99, 211, 200, 204, 205, 85, 211, 245, 15, 126, + /* 420 */ 245, 249, 245, 211, 249, 245, 249, 206, 85, 249, + /* 430 */ 99, 209, 210, 85, 213, 119, 120, 118, 84, 245, + /* 440 */ 242, 245, 99, 249, 135, 249, 122, 99, 85, 85, + /* 450 */ 139, 85, 122, 85, 85, 85, 85, 1, 62, 85, + /* 460 */ 126, 85, 99, 99, 85, 99, 84, 99, 99, 99, + /* 470 */ 99, 117, 141, 99, 147, 99, 149, 267, 99, 147, + /* 480 */ 84, 149, 147, 267, 149, 5, 147, 7, 149, 147, + /* 490 */ 267, 149, 147, 267, 149, 39, 80, 81, 116, 267, /* 500 */ 200, 267, 267, 267, 267, 267, 267, 250, 267, 267, - /* 510 */ 267, 250, 240, 240, 240, 240, 240, 240, 200, 266, - /* 520 */ 276, 276, 252, 200, 200, 62, 200, 248, 200, 248, - /* 530 */ 248, 200, 91, 271, 271, 200, 271, 119, 200, 200, - /* 540 */ 262, 200, 200, 263, 200, 260, 200, 200, 200, 126, - /* 550 */ 200, 200, 200, 261, 200, 271, 200, 200, 200, 200, - /* 560 */ 200, 200, 200, 200, 138, 140, 133, 200, 137, 200, - /* 570 */ 200, 136, 259, 131, 200, 200, 130, 200, 200, 200, - /* 580 */ 200, 200, 129, 200, 200, 200, 200, 200, 200, 200, - /* 590 */ 200, 200, 200, 200, 200, 200, 200, 132, 200, 200, - /* 600 */ 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, - /* 610 */ 200, 200, 200, 200, 200, 115, 142, 98, 202, 202, - /* 620 */ 97, 202, 202, 202, 53, 94, 96, 57, 95, 93, - /* 630 */ 202, 86, 202, 5, 155, 5, 202, 5, 155, 5, - /* 640 */ 5, 102, 202, 101, 122, 144, 202, 208, 212, 212, - /* 650 */ 84, 208, 117, 85, 99, 123, 84, 99, 85, 202, - /* 660 */ 202, 84, 202, 215, 214, 203, 221, 220, 219, 218, - /* 670 */ 216, 203, 203, 217, 202, 209, 85, 239, 203, 84, - /* 680 */ 258, 257, 223, 256, 255, 253, 202, 202, 85, 99, - /* 690 */ 204, 239, 84, 1, 85, 84, 84, 99, 134, 99, - /* 700 */ 84, 134, 84, 117, 84, 118, 80, 5, 89, 88, - /* 710 */ 72, 89, 88, 9, 5, 5, 5, 5, 5, 5, - /* 720 */ 5, 15, 80, 87, 84, 119, 85, 84, 61, 26, - /* 730 */ 149, 149, 16, 16, 149, 5, 99, 149, 5, 85, - /* 740 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 750 */ 5, 5, 5, 5, 5, 99, 87, 62, 0, 279, - /* 760 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, - /* 770 */ 279, 21, 21, 279, 279, 279, 279, 279, 279, 279, + /* 510 */ 267, 267, 250, 250, 240, 248, 240, 240, 240, 240, + /* 520 */ 240, 240, 119, 200, 266, 200, 200, 276, 200, 200, + /* 530 */ 248, 276, 248, 200, 248, 62, 200, 200, 200, 252, + /* 540 */ 200, 200, 253, 271, 200, 260, 200, 200, 126, 200, + /* 550 */ 200, 200, 200, 200, 263, 200, 200, 200, 200, 200, + /* 560 */ 200, 200, 200, 271, 138, 200, 271, 200, 271, 140, + /* 570 */ 262, 133, 200, 200, 261, 200, 137, 200, 200, 136, + /* 580 */ 200, 200, 131, 200, 200, 200, 200, 200, 200, 200, + /* 590 */ 200, 200, 200, 200, 200, 200, 130, 200, 200, 200, + /* 600 */ 200, 200, 200, 129, 200, 200, 200, 200, 200, 200, + /* 610 */ 200, 200, 200, 200, 200, 200, 132, 200, 142, 91, + /* 620 */ 202, 202, 202, 202, 115, 202, 53, 98, 97, 94, + /* 630 */ 96, 57, 202, 95, 202, 202, 93, 86, 5, 5, + /* 640 */ 5, 155, 155, 202, 202, 208, 212, 212, 208, 5, + /* 650 */ 5, 102, 144, 101, 122, 84, 202, 117, 99, 202, + /* 660 */ 85, 203, 215, 203, 219, 221, 220, 218, 216, 214, + /* 670 */ 217, 202, 202, 223, 203, 239, 209, 84, 203, 202, + /* 680 */ 202, 99, 257, 259, 256, 258, 255, 239, 85, 204, + /* 690 */ 84, 99, 85, 84, 1, 85, 84, 99, 85, 84, + /* 700 */ 84, 134, 134, 99, 84, 84, 117, 84, 118, 80, + /* 710 */ 72, 89, 88, 5, 89, 88, 9, 5, 5, 5, + /* 720 */ 5, 5, 5, 5, 87, 15, 80, 84, 26, 61, + /* 730 */ 119, 99, 85, 84, 149, 16, 149, 16, 5, 5, + /* 740 */ 149, 85, 5, 149, 5, 5, 5, 5, 5, 5, + /* 750 */ 5, 5, 5, 5, 5, 5, 5, 5, 99, 87, + /* 760 */ 62, 0, 279, 279, 279, 279, 279, 279, 279, 279, + /* 770 */ 279, 279, 279, 279, 21, 21, 279, 279, 279, 279, /* 780 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, /* 790 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, /* 800 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, @@ -384,113 +384,114 @@ static const YYCODETYPE yy_lookahead[] = { /* 940 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, /* 950 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, /* 960 */ 279, 279, 279, 279, 279, 279, 279, 279, 279, 279, + /* 970 */ 279, 279, 279, }; #define YY_SHIFT_COUNT (367) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (758) +#define YY_SHIFT_MAX (761) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 186, 113, 113, 260, 260, 17, 250, 266, 266, 193, + /* 0 */ 186, 113, 113, 260, 260, 98, 250, 266, 266, 193, /* 10 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, /* 20 */ 23, 23, 23, 0, 122, 266, 316, 316, 316, 9, - /* 30 */ 9, 23, 23, 18, 23, 78, 23, 23, 23, 23, - /* 40 */ 174, 17, 92, 92, 48, 773, 773, 773, 266, 266, + /* 30 */ 9, 23, 23, 18, 23, 204, 23, 23, 23, 23, + /* 40 */ 174, 98, 166, 166, 48, 776, 776, 776, 266, 266, /* 50 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, /* 60 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 70 */ 316, 316, 316, 2, 2, 2, 2, 2, 2, 2, - /* 80 */ 23, 23, 23, 27, 23, 23, 23, 9, 9, 23, - /* 90 */ 23, 23, 23, 288, 288, 151, 9, 23, 23, 23, + /* 70 */ 316, 316, 316, 80, 2, 2, 2, 2, 2, 2, + /* 80 */ 2, 23, 23, 23, 64, 23, 23, 23, 9, 9, + /* 90 */ 23, 23, 23, 23, 309, 309, 151, 9, 23, 23, /* 100 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, /* 110 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, /* 120 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, /* 130 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, /* 140 */ 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - /* 150 */ 23, 23, 23, 23, 23, 23, 463, 463, 463, 423, - /* 160 */ 423, 423, 423, 463, 463, 426, 425, 433, 431, 435, - /* 170 */ 442, 446, 453, 465, 474, 463, 463, 463, 441, 441, - /* 180 */ 500, 17, 17, 463, 463, 519, 523, 571, 531, 530, - /* 190 */ 570, 533, 536, 500, 48, 463, 463, 545, 545, 463, - /* 200 */ 545, 463, 545, 463, 463, 773, 773, 29, 69, 69, - /* 210 */ 99, 69, 127, 170, 240, 252, 252, 252, 252, 252, - /* 220 */ 252, 271, 283, 40, 40, 40, 40, 233, 247, 130, - /* 230 */ 319, 269, 269, 293, 334, 322, 164, 333, 320, 325, - /* 240 */ 345, 349, 353, 332, 307, 355, 364, 365, 366, 368, - /* 250 */ 371, 372, 373, 455, 397, 418, 376, 321, 327, 330, - /* 260 */ 473, 477, 336, 340, 382, 343, 415, 628, 479, 630, - /* 270 */ 632, 483, 634, 635, 539, 542, 501, 522, 535, 566, - /* 280 */ 532, 568, 572, 555, 558, 573, 577, 591, 595, 603, - /* 290 */ 590, 608, 609, 611, 692, 612, 598, 564, 600, 567, - /* 300 */ 616, 535, 618, 586, 620, 587, 626, 619, 621, 638, - /* 310 */ 702, 622, 624, 704, 709, 710, 711, 712, 713, 714, - /* 320 */ 715, 636, 706, 642, 640, 641, 606, 643, 703, 667, - /* 330 */ 716, 581, 582, 637, 637, 637, 637, 717, 585, 588, - /* 340 */ 637, 637, 637, 730, 733, 654, 637, 735, 736, 737, - /* 350 */ 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, - /* 360 */ 748, 749, 656, 669, 750, 751, 695, 758, + /* 150 */ 23, 23, 23, 23, 23, 23, 23, 473, 473, 473, + /* 160 */ 422, 422, 422, 422, 473, 473, 426, 429, 438, 439, + /* 170 */ 443, 451, 466, 474, 484, 476, 473, 473, 473, 528, + /* 180 */ 528, 509, 98, 98, 473, 473, 529, 531, 573, 535, + /* 190 */ 534, 574, 538, 543, 509, 48, 473, 473, 551, 551, + /* 200 */ 473, 551, 473, 551, 473, 473, 776, 776, 29, 69, + /* 210 */ 69, 99, 69, 127, 170, 240, 252, 252, 252, 252, + /* 220 */ 252, 252, 271, 283, 40, 40, 40, 40, 233, 247, + /* 230 */ 130, 319, 269, 269, 293, 334, 322, 164, 343, 324, + /* 240 */ 330, 348, 363, 364, 331, 311, 366, 368, 369, 370, + /* 250 */ 371, 354, 374, 376, 456, 396, 403, 379, 327, 332, + /* 260 */ 335, 317, 480, 339, 342, 382, 345, 416, 633, 486, + /* 270 */ 634, 635, 487, 644, 645, 549, 552, 508, 532, 540, + /* 280 */ 571, 575, 593, 559, 582, 603, 606, 607, 609, 610, + /* 290 */ 592, 612, 613, 615, 693, 616, 598, 567, 604, 568, + /* 300 */ 620, 540, 621, 589, 623, 590, 629, 622, 624, 638, + /* 310 */ 708, 625, 627, 707, 712, 713, 714, 715, 716, 717, + /* 320 */ 718, 637, 710, 646, 643, 647, 611, 649, 702, 668, + /* 330 */ 719, 585, 587, 632, 632, 632, 632, 721, 591, 594, + /* 340 */ 632, 632, 632, 733, 734, 656, 632, 737, 739, 740, + /* 350 */ 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, + /* 360 */ 751, 752, 659, 672, 753, 754, 698, 761, }; -#define YY_REDUCE_COUNT (206) +#define YY_REDUCE_COUNT (207) #define YY_REDUCE_MIN (-267) -#define YY_REDUCE_MAX (486) +#define YY_REDUCE_MAX (485) static const short yy_reduce_ofst[] = { /* 0 */ -187, 10, 10, -164, -164, 53, -160, 65, 121, -169, /* 10 */ -113, -116, -42, 116, 119, 133, 159, 172, 175, 177, /* 20 */ 180, 194, 196, -196, -198, -259, -240, -202, -144, -230, - /* 30 */ -131, -63, 3, -239, -58, -43, 200, 201, 202, -89, - /* 40 */ 47, 82, 114, 203, -15, 75, 218, 210, -267, -262, - /* 50 */ -79, -9, 32, 55, 120, 195, 219, 224, 226, 230, - /* 60 */ 232, 234, 235, 236, 237, 238, 239, 241, 242, 243, - /* 70 */ -46, 257, 261, 229, 272, 273, 274, 275, 276, 277, - /* 80 */ 300, 318, 323, 253, 324, 326, 328, 279, 281, 331, - /* 90 */ 335, 338, 339, 244, 245, 270, 282, 341, 342, 344, - /* 100 */ 346, 347, 348, 350, 351, 352, 354, 356, 357, 358, - /* 110 */ 359, 360, 361, 362, 363, 367, 369, 370, 374, 375, - /* 120 */ 377, 378, 379, 380, 381, 383, 384, 385, 386, 387, - /* 130 */ 388, 389, 390, 391, 392, 393, 394, 395, 396, 398, - /* 140 */ 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, - /* 150 */ 409, 410, 411, 412, 413, 414, 416, 417, 419, 262, - /* 160 */ 263, 265, 284, 420, 421, 280, 278, 292, 285, 313, - /* 170 */ 422, 424, 427, 429, 432, 428, 430, 434, 436, 437, - /* 180 */ 438, 439, 443, 440, 444, 445, 447, 449, 448, 451, - /* 190 */ 454, 456, 450, 452, 459, 457, 458, 462, 468, 460, - /* 200 */ 469, 472, 475, 484, 485, 466, 486, + /* 30 */ -131, -63, 126, -239, -58, 158, 200, 205, 212, -89, + /* 40 */ 47, 198, 114, 221, -15, -100, 222, 209, -267, -262, + /* 50 */ -201, -189, 11, 32, 73, 210, 216, 223, 226, 232, + /* 60 */ 234, 235, 236, 237, 238, 239, 241, 242, 243, 244, + /* 70 */ 257, 262, 263, 267, 274, 276, 277, 278, 279, 280, + /* 80 */ 281, 300, 323, 325, 258, 326, 328, 329, 282, 284, + /* 90 */ 333, 336, 337, 338, 251, 255, 287, 286, 340, 341, + /* 100 */ 344, 346, 347, 349, 350, 351, 352, 353, 355, 356, + /* 110 */ 357, 358, 359, 360, 361, 362, 365, 367, 372, 373, + /* 120 */ 375, 377, 378, 380, 381, 383, 384, 385, 386, 387, + /* 130 */ 388, 389, 390, 391, 392, 393, 394, 395, 397, 398, + /* 140 */ 399, 400, 401, 402, 404, 405, 406, 407, 408, 409, + /* 150 */ 410, 411, 412, 413, 414, 415, 417, 418, 419, 420, + /* 160 */ 272, 292, 295, 297, 421, 423, 291, 308, 313, 285, + /* 170 */ 424, 427, 425, 428, 431, 289, 430, 432, 433, 434, + /* 180 */ 435, 436, 437, 440, 441, 442, 444, 446, 445, 447, + /* 190 */ 449, 452, 453, 455, 448, 450, 454, 457, 458, 460, + /* 200 */ 469, 471, 470, 475, 477, 478, 467, 485, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 870, 994, 933, 1004, 920, 930, 1145, 1145, 1145, 870, - /* 10 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 20 */ 870, 870, 870, 1052, 890, 1145, 870, 870, 870, 870, - /* 30 */ 870, 870, 870, 1067, 870, 930, 870, 870, 870, 870, - /* 40 */ 940, 930, 940, 940, 870, 1047, 978, 996, 870, 870, - /* 50 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 60 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 70 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 80 */ 870, 870, 870, 1054, 1060, 1057, 870, 870, 870, 1062, - /* 90 */ 870, 870, 870, 1086, 1086, 1045, 870, 870, 870, 870, - /* 100 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 110 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 120 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 918, - /* 130 */ 870, 916, 870, 870, 870, 870, 870, 870, 870, 870, - /* 140 */ 870, 870, 870, 870, 870, 870, 870, 870, 901, 870, - /* 150 */ 870, 870, 870, 870, 870, 888, 892, 892, 892, 870, - /* 160 */ 870, 870, 870, 892, 892, 1093, 1097, 1079, 1091, 1087, - /* 170 */ 1074, 1072, 1070, 1078, 1101, 892, 892, 892, 938, 938, - /* 180 */ 934, 930, 930, 892, 892, 956, 954, 952, 944, 950, - /* 190 */ 946, 948, 942, 921, 870, 892, 892, 928, 928, 892, - /* 200 */ 928, 892, 928, 892, 892, 978, 996, 870, 1102, 1092, - /* 210 */ 870, 1144, 1132, 1131, 870, 1140, 1139, 1138, 1130, 1129, - /* 220 */ 1128, 870, 870, 1124, 1127, 1126, 1125, 870, 870, 870, - /* 230 */ 870, 1134, 1133, 870, 870, 870, 870, 870, 870, 870, - /* 240 */ 870, 870, 870, 1098, 1094, 870, 870, 870, 870, 870, - /* 250 */ 870, 870, 870, 870, 1104, 870, 870, 870, 870, 870, - /* 260 */ 870, 870, 870, 870, 1006, 870, 870, 870, 870, 870, - /* 270 */ 870, 870, 870, 870, 870, 870, 870, 1044, 870, 870, - /* 280 */ 870, 870, 870, 1056, 1055, 870, 870, 870, 870, 870, - /* 290 */ 870, 870, 870, 870, 870, 870, 1088, 870, 1080, 870, - /* 300 */ 870, 1018, 870, 870, 870, 870, 870, 870, 870, 870, - /* 310 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 320 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 330 */ 870, 870, 870, 1163, 1158, 1159, 1156, 870, 870, 870, - /* 340 */ 1155, 1150, 1151, 870, 870, 870, 1148, 870, 870, 870, - /* 350 */ 870, 870, 870, 870, 870, 870, 870, 870, 870, 870, - /* 360 */ 870, 870, 962, 870, 899, 897, 870, 870, + /* 0 */ 872, 996, 935, 1006, 922, 932, 1148, 1148, 1148, 872, + /* 10 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 20 */ 872, 872, 872, 1055, 892, 1148, 872, 872, 872, 872, + /* 30 */ 872, 872, 872, 1070, 872, 932, 872, 872, 872, 872, + /* 40 */ 942, 932, 942, 942, 872, 1050, 980, 998, 872, 872, + /* 50 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 60 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 70 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 80 */ 872, 872, 872, 872, 1057, 1063, 1060, 872, 872, 872, + /* 90 */ 1065, 872, 872, 872, 1089, 1089, 1048, 872, 872, 872, + /* 100 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 110 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 120 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 130 */ 920, 872, 918, 872, 872, 872, 872, 872, 872, 872, + /* 140 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 903, + /* 150 */ 872, 872, 872, 872, 872, 872, 890, 894, 894, 894, + /* 160 */ 872, 872, 872, 872, 894, 894, 1096, 1100, 1082, 1094, + /* 170 */ 1090, 1077, 1075, 1073, 1081, 1104, 894, 894, 894, 940, + /* 180 */ 940, 936, 932, 932, 894, 894, 958, 956, 954, 946, + /* 190 */ 952, 948, 950, 944, 923, 872, 894, 894, 930, 930, + /* 200 */ 894, 930, 894, 930, 894, 894, 980, 998, 872, 1105, + /* 210 */ 1095, 872, 1147, 1135, 1134, 872, 1143, 1142, 1141, 1133, + /* 220 */ 1132, 1131, 872, 872, 1127, 1130, 1129, 1128, 872, 872, + /* 230 */ 872, 872, 1137, 1136, 872, 872, 872, 872, 872, 872, + /* 240 */ 872, 872, 872, 872, 1101, 1097, 872, 872, 872, 872, + /* 250 */ 872, 872, 872, 872, 872, 1107, 872, 872, 872, 872, + /* 260 */ 872, 872, 872, 872, 872, 1008, 872, 872, 872, 872, + /* 270 */ 872, 872, 872, 872, 872, 872, 872, 872, 1047, 872, + /* 280 */ 872, 872, 872, 1059, 1058, 872, 872, 872, 872, 872, + /* 290 */ 872, 872, 872, 872, 872, 872, 1091, 872, 1083, 872, + /* 300 */ 872, 1020, 872, 872, 872, 872, 872, 872, 872, 872, + /* 310 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 320 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 330 */ 872, 872, 872, 1166, 1161, 1162, 1159, 872, 872, 872, + /* 340 */ 1158, 1153, 1154, 872, 872, 872, 1151, 872, 872, 872, + /* 350 */ 872, 872, 872, 872, 872, 872, 872, 872, 872, 872, + /* 360 */ 872, 872, 964, 872, 901, 899, 872, 872, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1250,129 +1251,130 @@ static const char *const yyRuleName[] = { /* 168 */ "select ::= LP select RP", /* 169 */ "union ::= select", /* 170 */ "union ::= union UNION ALL select", - /* 171 */ "cmd ::= union", - /* 172 */ "select ::= SELECT selcollist", - /* 173 */ "sclp ::= selcollist COMMA", - /* 174 */ "sclp ::=", - /* 175 */ "selcollist ::= sclp distinct expr as", - /* 176 */ "selcollist ::= sclp STAR", - /* 177 */ "as ::= AS ids", - /* 178 */ "as ::= ids", - /* 179 */ "as ::=", - /* 180 */ "distinct ::= DISTINCT", - /* 181 */ "distinct ::=", - /* 182 */ "from ::= FROM tablelist", - /* 183 */ "from ::= FROM sub", - /* 184 */ "sub ::= LP union RP", - /* 185 */ "sub ::= LP union RP ids", - /* 186 */ "sub ::= sub COMMA LP union RP ids", - /* 187 */ "tablelist ::= ids cpxName", - /* 188 */ "tablelist ::= ids cpxName ids", - /* 189 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 190 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 191 */ "tmvar ::= VARIABLE", - /* 192 */ "interval_option ::= intervalKey LP tmvar RP", - /* 193 */ "interval_option ::= intervalKey LP tmvar COMMA tmvar RP", - /* 194 */ "interval_option ::=", - /* 195 */ "intervalKey ::= INTERVAL", - /* 196 */ "intervalKey ::= EVERY", - /* 197 */ "session_option ::=", - /* 198 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", - /* 199 */ "windowstate_option ::=", - /* 200 */ "windowstate_option ::= STATE_WINDOW LP ids RP", - /* 201 */ "fill_opt ::=", - /* 202 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 203 */ "fill_opt ::= FILL LP ID RP", - /* 204 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 205 */ "sliding_opt ::=", - /* 206 */ "orderby_opt ::=", - /* 207 */ "orderby_opt ::= ORDER BY sortlist", - /* 208 */ "sortlist ::= sortlist COMMA item sortorder", - /* 209 */ "sortlist ::= item sortorder", - /* 210 */ "item ::= ids cpxName", - /* 211 */ "sortorder ::= ASC", - /* 212 */ "sortorder ::= DESC", - /* 213 */ "sortorder ::=", - /* 214 */ "groupby_opt ::=", - /* 215 */ "groupby_opt ::= GROUP BY grouplist", - /* 216 */ "grouplist ::= grouplist COMMA item", - /* 217 */ "grouplist ::= item", - /* 218 */ "having_opt ::=", - /* 219 */ "having_opt ::= HAVING expr", - /* 220 */ "limit_opt ::=", - /* 221 */ "limit_opt ::= LIMIT signed", - /* 222 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 223 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 224 */ "slimit_opt ::=", - /* 225 */ "slimit_opt ::= SLIMIT signed", - /* 226 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 227 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 228 */ "where_opt ::=", - /* 229 */ "where_opt ::= WHERE expr", - /* 230 */ "expr ::= LP expr RP", - /* 231 */ "expr ::= ID", - /* 232 */ "expr ::= ID DOT ID", - /* 233 */ "expr ::= ID DOT STAR", - /* 234 */ "expr ::= INTEGER", - /* 235 */ "expr ::= MINUS INTEGER", - /* 236 */ "expr ::= PLUS INTEGER", - /* 237 */ "expr ::= FLOAT", - /* 238 */ "expr ::= MINUS FLOAT", - /* 239 */ "expr ::= PLUS FLOAT", - /* 240 */ "expr ::= STRING", - /* 241 */ "expr ::= NOW", - /* 242 */ "expr ::= VARIABLE", - /* 243 */ "expr ::= PLUS VARIABLE", - /* 244 */ "expr ::= MINUS VARIABLE", - /* 245 */ "expr ::= BOOL", - /* 246 */ "expr ::= NULL", - /* 247 */ "expr ::= ID LP exprlist RP", - /* 248 */ "expr ::= ID LP STAR RP", - /* 249 */ "expr ::= expr IS NULL", - /* 250 */ "expr ::= expr IS NOT NULL", - /* 251 */ "expr ::= expr LT expr", - /* 252 */ "expr ::= expr GT expr", - /* 253 */ "expr ::= expr LE expr", - /* 254 */ "expr ::= expr GE expr", - /* 255 */ "expr ::= expr NE expr", - /* 256 */ "expr ::= expr EQ expr", - /* 257 */ "expr ::= expr BETWEEN expr AND expr", - /* 258 */ "expr ::= expr AND expr", - /* 259 */ "expr ::= expr OR expr", - /* 260 */ "expr ::= expr PLUS expr", - /* 261 */ "expr ::= expr MINUS expr", - /* 262 */ "expr ::= expr STAR expr", - /* 263 */ "expr ::= expr SLASH expr", - /* 264 */ "expr ::= expr REM expr", - /* 265 */ "expr ::= expr LIKE expr", - /* 266 */ "expr ::= expr MATCH expr", - /* 267 */ "expr ::= expr NMATCH expr", - /* 268 */ "expr ::= expr IN LP exprlist RP", - /* 269 */ "exprlist ::= exprlist COMMA expritem", - /* 270 */ "exprlist ::= expritem", - /* 271 */ "expritem ::= expr", - /* 272 */ "expritem ::=", - /* 273 */ "cmd ::= RESET QUERY CACHE", - /* 274 */ "cmd ::= SYNCDB ids REPLICA", - /* 275 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 276 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 277 */ "cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist", - /* 278 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 279 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 280 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 281 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 282 */ "cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist", - /* 283 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 284 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 285 */ "cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist", - /* 286 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 287 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 288 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 289 */ "cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem", - /* 290 */ "cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist", - /* 291 */ "cmd ::= KILL CONNECTION INTEGER", - /* 292 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 293 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 171 */ "union ::= union UNION select", + /* 172 */ "cmd ::= union", + /* 173 */ "select ::= SELECT selcollist", + /* 174 */ "sclp ::= selcollist COMMA", + /* 175 */ "sclp ::=", + /* 176 */ "selcollist ::= sclp distinct expr as", + /* 177 */ "selcollist ::= sclp STAR", + /* 178 */ "as ::= AS ids", + /* 179 */ "as ::= ids", + /* 180 */ "as ::=", + /* 181 */ "distinct ::= DISTINCT", + /* 182 */ "distinct ::=", + /* 183 */ "from ::= FROM tablelist", + /* 184 */ "from ::= FROM sub", + /* 185 */ "sub ::= LP union RP", + /* 186 */ "sub ::= LP union RP ids", + /* 187 */ "sub ::= sub COMMA LP union RP ids", + /* 188 */ "tablelist ::= ids cpxName", + /* 189 */ "tablelist ::= ids cpxName ids", + /* 190 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 191 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 192 */ "tmvar ::= VARIABLE", + /* 193 */ "interval_option ::= intervalKey LP tmvar RP", + /* 194 */ "interval_option ::= intervalKey LP tmvar COMMA tmvar RP", + /* 195 */ "interval_option ::=", + /* 196 */ "intervalKey ::= INTERVAL", + /* 197 */ "intervalKey ::= EVERY", + /* 198 */ "session_option ::=", + /* 199 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", + /* 200 */ "windowstate_option ::=", + /* 201 */ "windowstate_option ::= STATE_WINDOW LP ids RP", + /* 202 */ "fill_opt ::=", + /* 203 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 204 */ "fill_opt ::= FILL LP ID RP", + /* 205 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 206 */ "sliding_opt ::=", + /* 207 */ "orderby_opt ::=", + /* 208 */ "orderby_opt ::= ORDER BY sortlist", + /* 209 */ "sortlist ::= sortlist COMMA item sortorder", + /* 210 */ "sortlist ::= item sortorder", + /* 211 */ "item ::= ids cpxName", + /* 212 */ "sortorder ::= ASC", + /* 213 */ "sortorder ::= DESC", + /* 214 */ "sortorder ::=", + /* 215 */ "groupby_opt ::=", + /* 216 */ "groupby_opt ::= GROUP BY grouplist", + /* 217 */ "grouplist ::= grouplist COMMA item", + /* 218 */ "grouplist ::= item", + /* 219 */ "having_opt ::=", + /* 220 */ "having_opt ::= HAVING expr", + /* 221 */ "limit_opt ::=", + /* 222 */ "limit_opt ::= LIMIT signed", + /* 223 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 224 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 225 */ "slimit_opt ::=", + /* 226 */ "slimit_opt ::= SLIMIT signed", + /* 227 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 228 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 229 */ "where_opt ::=", + /* 230 */ "where_opt ::= WHERE expr", + /* 231 */ "expr ::= LP expr RP", + /* 232 */ "expr ::= ID", + /* 233 */ "expr ::= ID DOT ID", + /* 234 */ "expr ::= ID DOT STAR", + /* 235 */ "expr ::= INTEGER", + /* 236 */ "expr ::= MINUS INTEGER", + /* 237 */ "expr ::= PLUS INTEGER", + /* 238 */ "expr ::= FLOAT", + /* 239 */ "expr ::= MINUS FLOAT", + /* 240 */ "expr ::= PLUS FLOAT", + /* 241 */ "expr ::= STRING", + /* 242 */ "expr ::= NOW", + /* 243 */ "expr ::= VARIABLE", + /* 244 */ "expr ::= PLUS VARIABLE", + /* 245 */ "expr ::= MINUS VARIABLE", + /* 246 */ "expr ::= BOOL", + /* 247 */ "expr ::= NULL", + /* 248 */ "expr ::= ID LP exprlist RP", + /* 249 */ "expr ::= ID LP STAR RP", + /* 250 */ "expr ::= expr IS NULL", + /* 251 */ "expr ::= expr IS NOT NULL", + /* 252 */ "expr ::= expr LT expr", + /* 253 */ "expr ::= expr GT expr", + /* 254 */ "expr ::= expr LE expr", + /* 255 */ "expr ::= expr GE expr", + /* 256 */ "expr ::= expr NE expr", + /* 257 */ "expr ::= expr EQ expr", + /* 258 */ "expr ::= expr BETWEEN expr AND expr", + /* 259 */ "expr ::= expr AND expr", + /* 260 */ "expr ::= expr OR expr", + /* 261 */ "expr ::= expr PLUS expr", + /* 262 */ "expr ::= expr MINUS expr", + /* 263 */ "expr ::= expr STAR expr", + /* 264 */ "expr ::= expr SLASH expr", + /* 265 */ "expr ::= expr REM expr", + /* 266 */ "expr ::= expr LIKE expr", + /* 267 */ "expr ::= expr MATCH expr", + /* 268 */ "expr ::= expr NMATCH expr", + /* 269 */ "expr ::= expr IN LP exprlist RP", + /* 270 */ "exprlist ::= exprlist COMMA expritem", + /* 271 */ "exprlist ::= expritem", + /* 272 */ "expritem ::= expr", + /* 273 */ "expritem ::=", + /* 274 */ "cmd ::= RESET QUERY CACHE", + /* 275 */ "cmd ::= SYNCDB ids REPLICA", + /* 276 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 277 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 278 */ "cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist", + /* 279 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 280 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 281 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 282 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 283 */ "cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist", + /* 284 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 285 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 286 */ "cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist", + /* 287 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 288 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 289 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 290 */ "cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem", + /* 291 */ "cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist", + /* 292 */ "cmd ::= KILL CONNECTION INTEGER", + /* 293 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 294 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1541,7 +1543,7 @@ tSqlExprDestroy((yypminor->yy526)); break; case 264: /* union */ { -destroyAllSqlNode((yypminor->yy135)); +destroyAllSqlNode((yypminor->yy551)); } break; case 274: /* sortitem */ @@ -2011,129 +2013,130 @@ static const struct { { 248, -3 }, /* (168) select ::= LP select RP */ { 264, -1 }, /* (169) union ::= select */ { 264, -4 }, /* (170) union ::= union UNION ALL select */ - { 199, -1 }, /* (171) cmd ::= union */ - { 248, -2 }, /* (172) select ::= SELECT selcollist */ - { 265, -2 }, /* (173) sclp ::= selcollist COMMA */ - { 265, 0 }, /* (174) sclp ::= */ - { 251, -4 }, /* (175) selcollist ::= sclp distinct expr as */ - { 251, -2 }, /* (176) selcollist ::= sclp STAR */ - { 268, -2 }, /* (177) as ::= AS ids */ - { 268, -1 }, /* (178) as ::= ids */ - { 268, 0 }, /* (179) as ::= */ - { 266, -1 }, /* (180) distinct ::= DISTINCT */ - { 266, 0 }, /* (181) distinct ::= */ - { 252, -2 }, /* (182) from ::= FROM tablelist */ - { 252, -2 }, /* (183) from ::= FROM sub */ - { 270, -3 }, /* (184) sub ::= LP union RP */ - { 270, -4 }, /* (185) sub ::= LP union RP ids */ - { 270, -6 }, /* (186) sub ::= sub COMMA LP union RP ids */ - { 269, -2 }, /* (187) tablelist ::= ids cpxName */ - { 269, -3 }, /* (188) tablelist ::= ids cpxName ids */ - { 269, -4 }, /* (189) tablelist ::= tablelist COMMA ids cpxName */ - { 269, -5 }, /* (190) tablelist ::= tablelist COMMA ids cpxName ids */ - { 271, -1 }, /* (191) tmvar ::= VARIABLE */ - { 254, -4 }, /* (192) interval_option ::= intervalKey LP tmvar RP */ - { 254, -6 }, /* (193) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - { 254, 0 }, /* (194) interval_option ::= */ - { 272, -1 }, /* (195) intervalKey ::= INTERVAL */ - { 272, -1 }, /* (196) intervalKey ::= EVERY */ - { 256, 0 }, /* (197) session_option ::= */ - { 256, -7 }, /* (198) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - { 257, 0 }, /* (199) windowstate_option ::= */ - { 257, -4 }, /* (200) windowstate_option ::= STATE_WINDOW LP ids RP */ - { 258, 0 }, /* (201) fill_opt ::= */ - { 258, -6 }, /* (202) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 258, -4 }, /* (203) fill_opt ::= FILL LP ID RP */ - { 255, -4 }, /* (204) sliding_opt ::= SLIDING LP tmvar RP */ - { 255, 0 }, /* (205) sliding_opt ::= */ - { 261, 0 }, /* (206) orderby_opt ::= */ - { 261, -3 }, /* (207) orderby_opt ::= ORDER BY sortlist */ - { 273, -4 }, /* (208) sortlist ::= sortlist COMMA item sortorder */ - { 273, -2 }, /* (209) sortlist ::= item sortorder */ - { 275, -2 }, /* (210) item ::= ids cpxName */ - { 276, -1 }, /* (211) sortorder ::= ASC */ - { 276, -1 }, /* (212) sortorder ::= DESC */ - { 276, 0 }, /* (213) sortorder ::= */ - { 259, 0 }, /* (214) groupby_opt ::= */ - { 259, -3 }, /* (215) groupby_opt ::= GROUP BY grouplist */ - { 277, -3 }, /* (216) grouplist ::= grouplist COMMA item */ - { 277, -1 }, /* (217) grouplist ::= item */ - { 260, 0 }, /* (218) having_opt ::= */ - { 260, -2 }, /* (219) having_opt ::= HAVING expr */ - { 263, 0 }, /* (220) limit_opt ::= */ - { 263, -2 }, /* (221) limit_opt ::= LIMIT signed */ - { 263, -4 }, /* (222) limit_opt ::= LIMIT signed OFFSET signed */ - { 263, -4 }, /* (223) limit_opt ::= LIMIT signed COMMA signed */ - { 262, 0 }, /* (224) slimit_opt ::= */ - { 262, -2 }, /* (225) slimit_opt ::= SLIMIT signed */ - { 262, -4 }, /* (226) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 262, -4 }, /* (227) slimit_opt ::= SLIMIT signed COMMA signed */ - { 253, 0 }, /* (228) where_opt ::= */ - { 253, -2 }, /* (229) where_opt ::= WHERE expr */ - { 267, -3 }, /* (230) expr ::= LP expr RP */ - { 267, -1 }, /* (231) expr ::= ID */ - { 267, -3 }, /* (232) expr ::= ID DOT ID */ - { 267, -3 }, /* (233) expr ::= ID DOT STAR */ - { 267, -1 }, /* (234) expr ::= INTEGER */ - { 267, -2 }, /* (235) expr ::= MINUS INTEGER */ - { 267, -2 }, /* (236) expr ::= PLUS INTEGER */ - { 267, -1 }, /* (237) expr ::= FLOAT */ - { 267, -2 }, /* (238) expr ::= MINUS FLOAT */ - { 267, -2 }, /* (239) expr ::= PLUS FLOAT */ - { 267, -1 }, /* (240) expr ::= STRING */ - { 267, -1 }, /* (241) expr ::= NOW */ - { 267, -1 }, /* (242) expr ::= VARIABLE */ - { 267, -2 }, /* (243) expr ::= PLUS VARIABLE */ - { 267, -2 }, /* (244) expr ::= MINUS VARIABLE */ - { 267, -1 }, /* (245) expr ::= BOOL */ - { 267, -1 }, /* (246) expr ::= NULL */ - { 267, -4 }, /* (247) expr ::= ID LP exprlist RP */ - { 267, -4 }, /* (248) expr ::= ID LP STAR RP */ - { 267, -3 }, /* (249) expr ::= expr IS NULL */ - { 267, -4 }, /* (250) expr ::= expr IS NOT NULL */ - { 267, -3 }, /* (251) expr ::= expr LT expr */ - { 267, -3 }, /* (252) expr ::= expr GT expr */ - { 267, -3 }, /* (253) expr ::= expr LE expr */ - { 267, -3 }, /* (254) expr ::= expr GE expr */ - { 267, -3 }, /* (255) expr ::= expr NE expr */ - { 267, -3 }, /* (256) expr ::= expr EQ expr */ - { 267, -5 }, /* (257) expr ::= expr BETWEEN expr AND expr */ - { 267, -3 }, /* (258) expr ::= expr AND expr */ - { 267, -3 }, /* (259) expr ::= expr OR expr */ - { 267, -3 }, /* (260) expr ::= expr PLUS expr */ - { 267, -3 }, /* (261) expr ::= expr MINUS expr */ - { 267, -3 }, /* (262) expr ::= expr STAR expr */ - { 267, -3 }, /* (263) expr ::= expr SLASH expr */ - { 267, -3 }, /* (264) expr ::= expr REM expr */ - { 267, -3 }, /* (265) expr ::= expr LIKE expr */ - { 267, -3 }, /* (266) expr ::= expr MATCH expr */ - { 267, -3 }, /* (267) expr ::= expr NMATCH expr */ - { 267, -5 }, /* (268) expr ::= expr IN LP exprlist RP */ - { 207, -3 }, /* (269) exprlist ::= exprlist COMMA expritem */ - { 207, -1 }, /* (270) exprlist ::= expritem */ - { 278, -1 }, /* (271) expritem ::= expr */ - { 278, 0 }, /* (272) expritem ::= */ - { 199, -3 }, /* (273) cmd ::= RESET QUERY CACHE */ - { 199, -3 }, /* (274) cmd ::= SYNCDB ids REPLICA */ - { 199, -7 }, /* (275) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 199, -7 }, /* (276) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 199, -7 }, /* (277) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - { 199, -7 }, /* (278) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 199, -7 }, /* (279) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 199, -8 }, /* (280) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 199, -9 }, /* (281) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 199, -7 }, /* (282) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - { 199, -7 }, /* (283) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - { 199, -7 }, /* (284) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - { 199, -7 }, /* (285) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - { 199, -7 }, /* (286) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - { 199, -7 }, /* (287) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - { 199, -8 }, /* (288) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - { 199, -9 }, /* (289) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - { 199, -7 }, /* (290) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - { 199, -3 }, /* (291) cmd ::= KILL CONNECTION INTEGER */ - { 199, -5 }, /* (292) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 199, -5 }, /* (293) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 264, -3 }, /* (171) union ::= union UNION select */ + { 199, -1 }, /* (172) cmd ::= union */ + { 248, -2 }, /* (173) select ::= SELECT selcollist */ + { 265, -2 }, /* (174) sclp ::= selcollist COMMA */ + { 265, 0 }, /* (175) sclp ::= */ + { 251, -4 }, /* (176) selcollist ::= sclp distinct expr as */ + { 251, -2 }, /* (177) selcollist ::= sclp STAR */ + { 268, -2 }, /* (178) as ::= AS ids */ + { 268, -1 }, /* (179) as ::= ids */ + { 268, 0 }, /* (180) as ::= */ + { 266, -1 }, /* (181) distinct ::= DISTINCT */ + { 266, 0 }, /* (182) distinct ::= */ + { 252, -2 }, /* (183) from ::= FROM tablelist */ + { 252, -2 }, /* (184) from ::= FROM sub */ + { 270, -3 }, /* (185) sub ::= LP union RP */ + { 270, -4 }, /* (186) sub ::= LP union RP ids */ + { 270, -6 }, /* (187) sub ::= sub COMMA LP union RP ids */ + { 269, -2 }, /* (188) tablelist ::= ids cpxName */ + { 269, -3 }, /* (189) tablelist ::= ids cpxName ids */ + { 269, -4 }, /* (190) tablelist ::= tablelist COMMA ids cpxName */ + { 269, -5 }, /* (191) tablelist ::= tablelist COMMA ids cpxName ids */ + { 271, -1 }, /* (192) tmvar ::= VARIABLE */ + { 254, -4 }, /* (193) interval_option ::= intervalKey LP tmvar RP */ + { 254, -6 }, /* (194) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ + { 254, 0 }, /* (195) interval_option ::= */ + { 272, -1 }, /* (196) intervalKey ::= INTERVAL */ + { 272, -1 }, /* (197) intervalKey ::= EVERY */ + { 256, 0 }, /* (198) session_option ::= */ + { 256, -7 }, /* (199) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + { 257, 0 }, /* (200) windowstate_option ::= */ + { 257, -4 }, /* (201) windowstate_option ::= STATE_WINDOW LP ids RP */ + { 258, 0 }, /* (202) fill_opt ::= */ + { 258, -6 }, /* (203) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 258, -4 }, /* (204) fill_opt ::= FILL LP ID RP */ + { 255, -4 }, /* (205) sliding_opt ::= SLIDING LP tmvar RP */ + { 255, 0 }, /* (206) sliding_opt ::= */ + { 261, 0 }, /* (207) orderby_opt ::= */ + { 261, -3 }, /* (208) orderby_opt ::= ORDER BY sortlist */ + { 273, -4 }, /* (209) sortlist ::= sortlist COMMA item sortorder */ + { 273, -2 }, /* (210) sortlist ::= item sortorder */ + { 275, -2 }, /* (211) item ::= ids cpxName */ + { 276, -1 }, /* (212) sortorder ::= ASC */ + { 276, -1 }, /* (213) sortorder ::= DESC */ + { 276, 0 }, /* (214) sortorder ::= */ + { 259, 0 }, /* (215) groupby_opt ::= */ + { 259, -3 }, /* (216) groupby_opt ::= GROUP BY grouplist */ + { 277, -3 }, /* (217) grouplist ::= grouplist COMMA item */ + { 277, -1 }, /* (218) grouplist ::= item */ + { 260, 0 }, /* (219) having_opt ::= */ + { 260, -2 }, /* (220) having_opt ::= HAVING expr */ + { 263, 0 }, /* (221) limit_opt ::= */ + { 263, -2 }, /* (222) limit_opt ::= LIMIT signed */ + { 263, -4 }, /* (223) limit_opt ::= LIMIT signed OFFSET signed */ + { 263, -4 }, /* (224) limit_opt ::= LIMIT signed COMMA signed */ + { 262, 0 }, /* (225) slimit_opt ::= */ + { 262, -2 }, /* (226) slimit_opt ::= SLIMIT signed */ + { 262, -4 }, /* (227) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 262, -4 }, /* (228) slimit_opt ::= SLIMIT signed COMMA signed */ + { 253, 0 }, /* (229) where_opt ::= */ + { 253, -2 }, /* (230) where_opt ::= WHERE expr */ + { 267, -3 }, /* (231) expr ::= LP expr RP */ + { 267, -1 }, /* (232) expr ::= ID */ + { 267, -3 }, /* (233) expr ::= ID DOT ID */ + { 267, -3 }, /* (234) expr ::= ID DOT STAR */ + { 267, -1 }, /* (235) expr ::= INTEGER */ + { 267, -2 }, /* (236) expr ::= MINUS INTEGER */ + { 267, -2 }, /* (237) expr ::= PLUS INTEGER */ + { 267, -1 }, /* (238) expr ::= FLOAT */ + { 267, -2 }, /* (239) expr ::= MINUS FLOAT */ + { 267, -2 }, /* (240) expr ::= PLUS FLOAT */ + { 267, -1 }, /* (241) expr ::= STRING */ + { 267, -1 }, /* (242) expr ::= NOW */ + { 267, -1 }, /* (243) expr ::= VARIABLE */ + { 267, -2 }, /* (244) expr ::= PLUS VARIABLE */ + { 267, -2 }, /* (245) expr ::= MINUS VARIABLE */ + { 267, -1 }, /* (246) expr ::= BOOL */ + { 267, -1 }, /* (247) expr ::= NULL */ + { 267, -4 }, /* (248) expr ::= ID LP exprlist RP */ + { 267, -4 }, /* (249) expr ::= ID LP STAR RP */ + { 267, -3 }, /* (250) expr ::= expr IS NULL */ + { 267, -4 }, /* (251) expr ::= expr IS NOT NULL */ + { 267, -3 }, /* (252) expr ::= expr LT expr */ + { 267, -3 }, /* (253) expr ::= expr GT expr */ + { 267, -3 }, /* (254) expr ::= expr LE expr */ + { 267, -3 }, /* (255) expr ::= expr GE expr */ + { 267, -3 }, /* (256) expr ::= expr NE expr */ + { 267, -3 }, /* (257) expr ::= expr EQ expr */ + { 267, -5 }, /* (258) expr ::= expr BETWEEN expr AND expr */ + { 267, -3 }, /* (259) expr ::= expr AND expr */ + { 267, -3 }, /* (260) expr ::= expr OR expr */ + { 267, -3 }, /* (261) expr ::= expr PLUS expr */ + { 267, -3 }, /* (262) expr ::= expr MINUS expr */ + { 267, -3 }, /* (263) expr ::= expr STAR expr */ + { 267, -3 }, /* (264) expr ::= expr SLASH expr */ + { 267, -3 }, /* (265) expr ::= expr REM expr */ + { 267, -3 }, /* (266) expr ::= expr LIKE expr */ + { 267, -3 }, /* (267) expr ::= expr MATCH expr */ + { 267, -3 }, /* (268) expr ::= expr NMATCH expr */ + { 267, -5 }, /* (269) expr ::= expr IN LP exprlist RP */ + { 207, -3 }, /* (270) exprlist ::= exprlist COMMA expritem */ + { 207, -1 }, /* (271) exprlist ::= expritem */ + { 278, -1 }, /* (272) expritem ::= expr */ + { 278, 0 }, /* (273) expritem ::= */ + { 199, -3 }, /* (274) cmd ::= RESET QUERY CACHE */ + { 199, -3 }, /* (275) cmd ::= SYNCDB ids REPLICA */ + { 199, -7 }, /* (276) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 199, -7 }, /* (277) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 199, -7 }, /* (278) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ + { 199, -7 }, /* (279) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 199, -7 }, /* (280) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 199, -8 }, /* (281) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 199, -9 }, /* (282) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 199, -7 }, /* (283) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ + { 199, -7 }, /* (284) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + { 199, -7 }, /* (285) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + { 199, -7 }, /* (286) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ + { 199, -7 }, /* (287) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + { 199, -7 }, /* (288) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + { 199, -8 }, /* (289) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + { 199, -9 }, /* (290) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ + { 199, -7 }, /* (291) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ + { 199, -3 }, /* (292) cmd ::= KILL CONNECTION INTEGER */ + { 199, -5 }, /* (293) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 199, -5 }, /* (294) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2414,7 +2417,7 @@ static void yy_reduce( break; case 55: /* ifexists ::= */ case 57: /* ifnotexists ::= */ yytestcase(yyruleno==57); - case 181: /* distinct ::= */ yytestcase(yyruleno==181); + case 182: /* distinct ::= */ yytestcase(yyruleno==182); { yymsp[1].minor.yy0.n = 0;} break; case 56: /* ifnotexists ::= IF NOT EXISTS */ @@ -2754,140 +2757,144 @@ static void yy_reduce( {yymsp[-2].minor.yy488 = yymsp[-1].minor.yy488;} break; case 169: /* union ::= select */ -{ yylhsminor.yy135 = setSubclause(NULL, yymsp[0].minor.yy488); } - yymsp[0].minor.yy135 = yylhsminor.yy135; +{ yylhsminor.yy551 = setSubclause(NULL, yymsp[0].minor.yy488); } + yymsp[0].minor.yy551 = yylhsminor.yy551; break; case 170: /* union ::= union UNION ALL select */ -{ yylhsminor.yy135 = appendSelectClause(yymsp[-3].minor.yy135, yymsp[0].minor.yy488); } - yymsp[-3].minor.yy135 = yylhsminor.yy135; +{ yylhsminor.yy551 = appendSelectClause(yymsp[-3].minor.yy551, SQL_TYPE_UNIONALL, yymsp[0].minor.yy488); } + yymsp[-3].minor.yy551 = yylhsminor.yy551; break; - case 171: /* cmd ::= union */ -{ setSqlInfo(pInfo, yymsp[0].minor.yy135, NULL, TSDB_SQL_SELECT); } + case 171: /* union ::= union UNION select */ +{ yylhsminor.yy551 = appendSelectClause(yymsp[-2].minor.yy551, SQL_TYPE_UNION, yymsp[0].minor.yy488); } + yymsp[-2].minor.yy551 = yylhsminor.yy551; break; - case 172: /* select ::= SELECT selcollist */ + case 172: /* cmd ::= union */ +{ setSqlInfo(pInfo, yymsp[0].minor.yy551, NULL, TSDB_SQL_SELECT); } + break; + case 173: /* select ::= SELECT selcollist */ { yylhsminor.yy488 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy135, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } yymsp[-1].minor.yy488 = yylhsminor.yy488; break; - case 173: /* sclp ::= selcollist COMMA */ + case 174: /* sclp ::= selcollist COMMA */ {yylhsminor.yy135 = yymsp[-1].minor.yy135;} yymsp[-1].minor.yy135 = yylhsminor.yy135; break; - case 174: /* sclp ::= */ - case 206: /* orderby_opt ::= */ yytestcase(yyruleno==206); + case 175: /* sclp ::= */ + case 207: /* orderby_opt ::= */ yytestcase(yyruleno==207); {yymsp[1].minor.yy135 = 0;} break; - case 175: /* selcollist ::= sclp distinct expr as */ + case 176: /* selcollist ::= sclp distinct expr as */ { yylhsminor.yy135 = tSqlExprListAppend(yymsp[-3].minor.yy135, yymsp[-1].minor.yy526, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } yymsp[-3].minor.yy135 = yylhsminor.yy135; break; - case 176: /* selcollist ::= sclp STAR */ + case 177: /* selcollist ::= sclp STAR */ { tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); yylhsminor.yy135 = tSqlExprListAppend(yymsp[-1].minor.yy135, pNode, 0, 0); } yymsp[-1].minor.yy135 = yylhsminor.yy135; break; - case 177: /* as ::= AS ids */ + case 178: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 178: /* as ::= ids */ + case 179: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 179: /* as ::= */ + case 180: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 180: /* distinct ::= DISTINCT */ + case 181: /* distinct ::= DISTINCT */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 182: /* from ::= FROM tablelist */ - case 183: /* from ::= FROM sub */ yytestcase(yyruleno==183); + case 183: /* from ::= FROM tablelist */ + case 184: /* from ::= FROM sub */ yytestcase(yyruleno==184); {yymsp[-1].minor.yy460 = yymsp[0].minor.yy460;} break; - case 184: /* sub ::= LP union RP */ -{yymsp[-2].minor.yy460 = addSubquery(NULL, yymsp[-1].minor.yy135, NULL);} + case 185: /* sub ::= LP union RP */ +{yymsp[-2].minor.yy460 = addSubquery(NULL, yymsp[-1].minor.yy551, NULL);} break; - case 185: /* sub ::= LP union RP ids */ -{yymsp[-3].minor.yy460 = addSubquery(NULL, yymsp[-2].minor.yy135, &yymsp[0].minor.yy0);} + case 186: /* sub ::= LP union RP ids */ +{yymsp[-3].minor.yy460 = addSubquery(NULL, yymsp[-2].minor.yy551, &yymsp[0].minor.yy0);} break; - case 186: /* sub ::= sub COMMA LP union RP ids */ -{yylhsminor.yy460 = addSubquery(yymsp[-5].minor.yy460, yymsp[-2].minor.yy135, &yymsp[0].minor.yy0);} + case 187: /* sub ::= sub COMMA LP union RP ids */ +{yylhsminor.yy460 = addSubquery(yymsp[-5].minor.yy460, yymsp[-2].minor.yy551, &yymsp[0].minor.yy0);} yymsp[-5].minor.yy460 = yylhsminor.yy460; break; - case 187: /* tablelist ::= ids cpxName */ + case 188: /* tablelist ::= ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy460 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); } yymsp[-1].minor.yy460 = yylhsminor.yy460; break; - case 188: /* tablelist ::= ids cpxName ids */ + case 189: /* tablelist ::= ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; yylhsminor.yy460 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } yymsp[-2].minor.yy460 = yylhsminor.yy460; break; - case 189: /* tablelist ::= tablelist COMMA ids cpxName */ + case 190: /* tablelist ::= tablelist COMMA ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy460 = setTableNameList(yymsp[-3].minor.yy460, &yymsp[-1].minor.yy0, NULL); } yymsp[-3].minor.yy460 = yylhsminor.yy460; break; - case 190: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 191: /* tablelist ::= tablelist COMMA ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; yylhsminor.yy460 = setTableNameList(yymsp[-4].minor.yy460, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } yymsp[-4].minor.yy460 = yylhsminor.yy460; break; - case 191: /* tmvar ::= VARIABLE */ + case 192: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 192: /* interval_option ::= intervalKey LP tmvar RP */ + case 193: /* interval_option ::= intervalKey LP tmvar RP */ {yylhsminor.yy160.interval = yymsp[-1].minor.yy0; yylhsminor.yy160.offset.n = 0; yylhsminor.yy160.token = yymsp[-3].minor.yy262;} yymsp[-3].minor.yy160 = yylhsminor.yy160; break; - case 193: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ + case 194: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ {yylhsminor.yy160.interval = yymsp[-3].minor.yy0; yylhsminor.yy160.offset = yymsp[-1].minor.yy0; yylhsminor.yy160.token = yymsp[-5].minor.yy262;} yymsp[-5].minor.yy160 = yylhsminor.yy160; break; - case 194: /* interval_option ::= */ + case 195: /* interval_option ::= */ {memset(&yymsp[1].minor.yy160, 0, sizeof(yymsp[1].minor.yy160));} break; - case 195: /* intervalKey ::= INTERVAL */ + case 196: /* intervalKey ::= INTERVAL */ {yymsp[0].minor.yy262 = TK_INTERVAL;} break; - case 196: /* intervalKey ::= EVERY */ + case 197: /* intervalKey ::= EVERY */ {yymsp[0].minor.yy262 = TK_EVERY; } break; - case 197: /* session_option ::= */ + case 198: /* session_option ::= */ {yymsp[1].minor.yy511.col.n = 0; yymsp[1].minor.yy511.gap.n = 0;} break; - case 198: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + case 199: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; yymsp[-6].minor.yy511.col = yymsp[-4].minor.yy0; yymsp[-6].minor.yy511.gap = yymsp[-1].minor.yy0; } break; - case 199: /* windowstate_option ::= */ + case 200: /* windowstate_option ::= */ { yymsp[1].minor.yy258.col.n = 0; yymsp[1].minor.yy258.col.z = NULL;} break; - case 200: /* windowstate_option ::= STATE_WINDOW LP ids RP */ + case 201: /* windowstate_option ::= STATE_WINDOW LP ids RP */ { yymsp[-3].minor.yy258.col = yymsp[-1].minor.yy0; } break; - case 201: /* fill_opt ::= */ + case 202: /* fill_opt ::= */ { yymsp[1].minor.yy135 = 0; } break; - case 202: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 203: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { SVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); @@ -2897,34 +2904,34 @@ static void yy_reduce( yymsp[-5].minor.yy135 = yymsp[-1].minor.yy135; } break; - case 203: /* fill_opt ::= FILL LP ID RP */ + case 204: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-3].minor.yy135 = tListItemAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 204: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 205: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 205: /* sliding_opt ::= */ + case 206: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 207: /* orderby_opt ::= ORDER BY sortlist */ + case 208: /* orderby_opt ::= ORDER BY sortlist */ {yymsp[-2].minor.yy135 = yymsp[0].minor.yy135;} break; - case 208: /* sortlist ::= sortlist COMMA item sortorder */ + case 209: /* sortlist ::= sortlist COMMA item sortorder */ { yylhsminor.yy135 = tListItemAppend(yymsp[-3].minor.yy135, &yymsp[-1].minor.yy191, yymsp[0].minor.yy130); } yymsp[-3].minor.yy135 = yylhsminor.yy135; break; - case 209: /* sortlist ::= item sortorder */ + case 210: /* sortlist ::= item sortorder */ { yylhsminor.yy135 = tListItemAppend(NULL, &yymsp[-1].minor.yy191, yymsp[0].minor.yy130); } yymsp[-1].minor.yy135 = yylhsminor.yy135; break; - case 210: /* item ::= ids cpxName */ + case 211: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; @@ -2933,260 +2940,258 @@ static void yy_reduce( } yymsp[-1].minor.yy191 = yylhsminor.yy191; break; - case 211: /* sortorder ::= ASC */ + case 212: /* sortorder ::= ASC */ { yymsp[0].minor.yy130 = TSDB_ORDER_ASC; } break; - case 212: /* sortorder ::= DESC */ + case 213: /* sortorder ::= DESC */ { yymsp[0].minor.yy130 = TSDB_ORDER_DESC;} break; - case 213: /* sortorder ::= */ + case 214: /* sortorder ::= */ { yymsp[1].minor.yy130 = TSDB_ORDER_ASC; } break; - case 214: /* groupby_opt ::= */ + case 215: /* groupby_opt ::= */ { yymsp[1].minor.yy135 = 0;} break; - case 215: /* groupby_opt ::= GROUP BY grouplist */ + case 216: /* groupby_opt ::= GROUP BY grouplist */ { yymsp[-2].minor.yy135 = yymsp[0].minor.yy135;} break; - case 216: /* grouplist ::= grouplist COMMA item */ + case 217: /* grouplist ::= grouplist COMMA item */ { yylhsminor.yy135 = tListItemAppend(yymsp[-2].minor.yy135, &yymsp[0].minor.yy191, -1); } yymsp[-2].minor.yy135 = yylhsminor.yy135; break; - case 217: /* grouplist ::= item */ + case 218: /* grouplist ::= item */ { yylhsminor.yy135 = tListItemAppend(NULL, &yymsp[0].minor.yy191, -1); } yymsp[0].minor.yy135 = yylhsminor.yy135; break; - case 218: /* having_opt ::= */ - case 228: /* where_opt ::= */ yytestcase(yyruleno==228); - case 272: /* expritem ::= */ yytestcase(yyruleno==272); + case 219: /* having_opt ::= */ + case 229: /* where_opt ::= */ yytestcase(yyruleno==229); + case 273: /* expritem ::= */ yytestcase(yyruleno==273); {yymsp[1].minor.yy526 = 0;} break; - case 219: /* having_opt ::= HAVING expr */ - case 229: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==229); + case 220: /* having_opt ::= HAVING expr */ + case 230: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==230); {yymsp[-1].minor.yy526 = yymsp[0].minor.yy526;} break; - case 220: /* limit_opt ::= */ - case 224: /* slimit_opt ::= */ yytestcase(yyruleno==224); + case 221: /* limit_opt ::= */ + case 225: /* slimit_opt ::= */ yytestcase(yyruleno==225); {yymsp[1].minor.yy247.limit = -1; yymsp[1].minor.yy247.offset = 0;} break; - case 221: /* limit_opt ::= LIMIT signed */ - case 225: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==225); + case 222: /* limit_opt ::= LIMIT signed */ + case 226: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==226); {yymsp[-1].minor.yy247.limit = yymsp[0].minor.yy531; yymsp[-1].minor.yy247.offset = 0;} break; - case 222: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 223: /* limit_opt ::= LIMIT signed OFFSET signed */ { yymsp[-3].minor.yy247.limit = yymsp[-2].minor.yy531; yymsp[-3].minor.yy247.offset = yymsp[0].minor.yy531;} break; - case 223: /* limit_opt ::= LIMIT signed COMMA signed */ + case 224: /* limit_opt ::= LIMIT signed COMMA signed */ { yymsp[-3].minor.yy247.limit = yymsp[0].minor.yy531; yymsp[-3].minor.yy247.offset = yymsp[-2].minor.yy531;} break; - case 226: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ + case 227: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ {yymsp[-3].minor.yy247.limit = yymsp[-2].minor.yy531; yymsp[-3].minor.yy247.offset = yymsp[0].minor.yy531;} break; - case 227: /* slimit_opt ::= SLIMIT signed COMMA signed */ + case 228: /* slimit_opt ::= SLIMIT signed COMMA signed */ {yymsp[-3].minor.yy247.limit = yymsp[0].minor.yy531; yymsp[-3].minor.yy247.offset = yymsp[-2].minor.yy531;} break; - case 230: /* expr ::= LP expr RP */ + case 231: /* expr ::= LP expr RP */ {yylhsminor.yy526 = yymsp[-1].minor.yy526; yylhsminor.yy526->exprToken.z = yymsp[-2].minor.yy0.z; yylhsminor.yy526->exprToken.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 231: /* expr ::= ID */ + case 232: /* expr ::= ID */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 232: /* expr ::= ID DOT ID */ + case 233: /* expr ::= ID DOT ID */ { yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 233: /* expr ::= ID DOT STAR */ + case 234: /* expr ::= ID DOT STAR */ { yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 234: /* expr ::= INTEGER */ + case 235: /* expr ::= INTEGER */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 235: /* expr ::= MINUS INTEGER */ - case 236: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==236); + case 236: /* expr ::= MINUS INTEGER */ + case 237: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==237); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} yymsp[-1].minor.yy526 = yylhsminor.yy526; break; - case 237: /* expr ::= FLOAT */ + case 238: /* expr ::= FLOAT */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 238: /* expr ::= MINUS FLOAT */ - case 239: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==239); + case 239: /* expr ::= MINUS FLOAT */ + case 240: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==240); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} yymsp[-1].minor.yy526 = yylhsminor.yy526; break; - case 240: /* expr ::= STRING */ + case 241: /* expr ::= STRING */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 241: /* expr ::= NOW */ + case 242: /* expr ::= NOW */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 242: /* expr ::= VARIABLE */ + case 243: /* expr ::= VARIABLE */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 243: /* expr ::= PLUS VARIABLE */ - case 244: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==244); + case 244: /* expr ::= PLUS VARIABLE */ + case 245: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==245); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} yymsp[-1].minor.yy526 = yylhsminor.yy526; break; - case 245: /* expr ::= BOOL */ + case 246: /* expr ::= BOOL */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 246: /* expr ::= NULL */ + case 247: /* expr ::= NULL */ { yylhsminor.yy526 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 247: /* expr ::= ID LP exprlist RP */ + case 248: /* expr ::= ID LP exprlist RP */ { tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy526 = tSqlExprCreateFunction(yymsp[-1].minor.yy135, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy526 = yylhsminor.yy526; break; - case 248: /* expr ::= ID LP STAR RP */ + case 249: /* expr ::= ID LP STAR RP */ { tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy526 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy526 = yylhsminor.yy526; break; - case 249: /* expr ::= expr IS NULL */ + case 250: /* expr ::= expr IS NULL */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, NULL, TK_ISNULL);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 250: /* expr ::= expr IS NOT NULL */ + case 251: /* expr ::= expr IS NOT NULL */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-3].minor.yy526, NULL, TK_NOTNULL);} yymsp[-3].minor.yy526 = yylhsminor.yy526; break; - case 251: /* expr ::= expr LT expr */ + case 252: /* expr ::= expr LT expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_LT);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 252: /* expr ::= expr GT expr */ + case 253: /* expr ::= expr GT expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_GT);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 253: /* expr ::= expr LE expr */ + case 254: /* expr ::= expr LE expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_LE);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 254: /* expr ::= expr GE expr */ + case 255: /* expr ::= expr GE expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_GE);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 255: /* expr ::= expr NE expr */ + case 256: /* expr ::= expr NE expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_NE);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 256: /* expr ::= expr EQ expr */ + case 257: /* expr ::= expr EQ expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_EQ);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 257: /* expr ::= expr BETWEEN expr AND expr */ + case 258: /* expr ::= expr BETWEEN expr AND expr */ { tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy526); yylhsminor.yy526 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy526, yymsp[-2].minor.yy526, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy526, TK_LE), TK_AND);} yymsp[-4].minor.yy526 = yylhsminor.yy526; break; - case 258: /* expr ::= expr AND expr */ + case 259: /* expr ::= expr AND expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_AND);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 259: /* expr ::= expr OR expr */ + case 260: /* expr ::= expr OR expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_OR); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 260: /* expr ::= expr PLUS expr */ + case 261: /* expr ::= expr PLUS expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_PLUS); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 261: /* expr ::= expr MINUS expr */ + case 262: /* expr ::= expr MINUS expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_MINUS); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 262: /* expr ::= expr STAR expr */ + case 263: /* expr ::= expr STAR expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_STAR); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 263: /* expr ::= expr SLASH expr */ + case 264: /* expr ::= expr SLASH expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_DIVIDE);} yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 264: /* expr ::= expr REM expr */ + case 265: /* expr ::= expr REM expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_REM); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 265: /* expr ::= expr LIKE expr */ + case 266: /* expr ::= expr LIKE expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_LIKE); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 266: /* expr ::= expr MATCH expr */ + case 267: /* expr ::= expr MATCH expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_MATCH); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 267: /* expr ::= expr NMATCH expr */ + case 268: /* expr ::= expr NMATCH expr */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-2].minor.yy526, yymsp[0].minor.yy526, TK_NMATCH); } yymsp[-2].minor.yy526 = yylhsminor.yy526; break; - case 268: /* expr ::= expr IN LP exprlist RP */ + case 269: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy526 = tSqlExprCreate(yymsp[-4].minor.yy526, (tSqlExpr*)yymsp[-1].minor.yy135, TK_IN); } yymsp[-4].minor.yy526 = yylhsminor.yy526; break; - case 269: /* exprlist ::= exprlist COMMA expritem */ + case 270: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy135 = tSqlExprListAppend(yymsp[-2].minor.yy135,yymsp[0].minor.yy526,0, 0);} yymsp[-2].minor.yy135 = yylhsminor.yy135; break; - case 270: /* exprlist ::= expritem */ + case 271: /* exprlist ::= expritem */ {yylhsminor.yy135 = tSqlExprListAppend(0,yymsp[0].minor.yy526,0, 0);} yymsp[0].minor.yy135 = yylhsminor.yy135; break; - case 271: /* expritem ::= expr */ + case 272: /* expritem ::= expr */ {yylhsminor.yy526 = yymsp[0].minor.yy526;} yymsp[0].minor.yy526 = yylhsminor.yy526; break; - case 273: /* cmd ::= RESET QUERY CACHE */ + case 274: /* cmd ::= RESET QUERY CACHE */ { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 274: /* cmd ::= SYNCDB ids REPLICA */ + case 275: /* cmd ::= SYNCDB ids REPLICA */ { setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} break; - case 275: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 276: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 276: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 277: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - toTSDBType(yymsp[0].minor.yy0.type); SArray* K = tListItemAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 277: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ + case 278: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 278: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 279: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 279: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 280: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3197,7 +3202,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 280: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 281: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3211,7 +3216,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 281: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 282: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -3223,21 +3228,21 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 282: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ + case 283: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 283: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + case 284: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 284: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + case 285: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3248,21 +3253,21 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 285: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ + case 286: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_CHANGE_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 286: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + case 287: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 287: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + case 288: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3273,7 +3278,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 288: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + case 289: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3287,7 +3292,7 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 289: /* cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ + case 290: /* cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -3299,20 +3304,20 @@ static void yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 290: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ + case 291: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy135, NULL, TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 291: /* cmd ::= KILL CONNECTION INTEGER */ + case 292: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 292: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 293: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 293: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 294: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 932613301b..6e491cedb0 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -75,7 +75,7 @@ void sqlCheck(const char* sql, bool valid) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -87,7 +87,7 @@ void sqlCheck(const char* sql, bool valid) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); if (valid) { @@ -112,7 +112,7 @@ TEST(testCase, validateAST_test) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -124,7 +124,7 @@ TEST(testCase, validateAST_test) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); SArray* pExprList = pQueryInfo->exprList[0]; @@ -170,7 +170,7 @@ TEST(testCase, function_Test) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -182,7 +182,7 @@ TEST(testCase, function_Test) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); SArray* pExprList = pQueryInfo->exprList[0]; @@ -216,7 +216,7 @@ TEST(testCase, function_Test2) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -228,7 +228,7 @@ TEST(testCase, function_Test2) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); SArray* pExprList = pQueryInfo->exprList[0]; @@ -262,7 +262,7 @@ TEST(testCase, function_Test3) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -274,7 +274,7 @@ TEST(testCase, function_Test3) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); SArray* pExprList = pQueryInfo->exprList[0]; @@ -307,7 +307,7 @@ TEST(testCase, function_Test4) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -319,7 +319,7 @@ TEST(testCase, function_Test4) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); SArray* pExprList = pQueryInfo->exprList[0]; @@ -355,7 +355,7 @@ TEST(testCase, function_Test5) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -367,7 +367,7 @@ TEST(testCase, function_Test5) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -410,6 +410,8 @@ TEST(testCase, function_Test10) { sqlCheck("select length(length(length(a))) from `t.1abc`", true); sqlCheck("select count() from `t.1abc`", false); sqlCheck("select block_dist() from `t.1abc`", true); + sqlCheck("select block_dist(a) from `t.1abc`", false); + sqlCheck("select count(*) from `t.1abc` interval(1s) group by a", false); } TEST(testCase, function_Test6) { @@ -422,7 +424,7 @@ TEST(testCase, function_Test6) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -434,7 +436,7 @@ TEST(testCase, function_Test6) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -492,7 +494,7 @@ TEST(testCase, function_Test6) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -504,7 +506,7 @@ TEST(testCase, function_Test6) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -551,7 +553,7 @@ TEST(testCase, function_Test6) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -563,7 +565,7 @@ TEST(testCase, function_Test6) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -601,7 +603,7 @@ TEST(testCase, function_Test6) { info1 = doGenerateAST("select sum(length(a)+length(b)) from `t.1abc` interval(10s, 1s)"); ASSERT_EQ(info1.valid, true); - pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -612,7 +614,7 @@ TEST(testCase, function_Test6) { pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -630,7 +632,7 @@ TEST(testCase, function_Test6) { SSqlInfo info1 = doGenerateAST("select count(k) from `t.1abc` interval(10s, 1s)"); ASSERT_EQ(info1.valid, true); - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -642,7 +644,7 @@ TEST(testCase, function_Test6) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_NE(ret, 0); @@ -653,7 +655,7 @@ TEST(testCase, function_Test6) { info1 = doGenerateAST("select top(a*b, ABC) from `t.1abc` interval(10s, 1s)"); ASSERT_EQ(info1.valid, true); - pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -664,7 +666,7 @@ TEST(testCase, function_Test6) { pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_NE(ret, 0); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index ded7d73e85..ee2c01dc48 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -75,7 +75,7 @@ void generateLogicplan(const char* sql) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -87,7 +87,7 @@ void generateLogicplan(const char* sql) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -115,7 +115,7 @@ TEST(testCase, planner_test) { buf.len = 128; buf.buf = msg; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf); ASSERT_EQ(code, 0); @@ -127,7 +127,7 @@ TEST(testCase, planner_test) { SQueryStmtInfo* pQueryInfo = createQueryInfo(); setTableMetaInfo(pQueryInfo, &req); - SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0); + SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.sub.node, 0); ret = validateSqlNode(pSqlNode, pQueryInfo, &buf); ASSERT_EQ(ret, 0); @@ -165,18 +165,18 @@ TEST(testCase, planner_test) { } TEST(testCase, displayPlan) { -// generateLogicplan("select count(*) from `t.1abc`"); -// generateLogicplan("select count(*)+ 22 from `t.1abc`"); -// generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30"); -// generateLogicplan("select count(*) from `t.1abc` group by a"); -// generateLogicplan("select count(A+B) from `t.1abc` group by a"); -// generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); -// generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); -// generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); -// generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); -// generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); -// generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20"); -// generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)"); + generateLogicplan("select count(*) from `t.1abc`"); + generateLogicplan("select count(*)+ 22 from `t.1abc`"); + generateLogicplan("select count(*)+ 22 from `t.1abc` interval(1h, 20s) sliding(10m) limit 20,30"); + generateLogicplan("select count(*) from `t.1abc` group by a"); + generateLogicplan("select count(A+B) from `t.1abc` group by a"); + generateLogicplan("select count(length(a)+b) from `t.1abc` group by a"); + generateLogicplan("select count(*) from `t.1abc` interval(10s, 5s) sliding(7s)"); + generateLogicplan("select count(*),sum(a),avg(b),min(a+b)+99 from `t.1abc`"); + generateLogicplan("select count(*), min(a) + 99 from `t.1abc`"); + generateLogicplan("select count(length(count(*) + 22)) from `t.1abc`"); + generateLogicplan("select concat(concat(a,b), concat(a,b)) from `t.1abc` limit 20"); + generateLogicplan("select count(*), first(a), last(b) from `t.1abc` state_window(a)"); generateLogicplan("select count(*), first(a), last(b) from `t.1abc` session(ts, 20s)"); // order by + group by column + limit offset + fill diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index 7d6f55692d..07ba46427f 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -680,12 +680,12 @@ TEST(testCase, generateAST_test) { msgBuf.buf = msg; msgBuf.len = 128; - SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0); + SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.sub.node), 0); int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &msgBuf); ASSERT_EQ(code, 0); SSqlInfo info2 = doGenerateAST("select * from abc where tsself, i, size); - if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -7981,11 +7981,11 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (pFromInfo->type == SQL_NODE_FROM_SUBQUERY){ + if (pFromInfo->type == SQL_FROM_NODE_SUBQUERY){ return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } - SRelElementPair* p1 = taosArrayGet(pFromInfo->list, 0); + SRelElement* p1 = taosArrayGet(pFromInfo->list, 0); SStrToken srcToken = {.z = p1->tableName.z, .n = p1->tableName.n, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -8415,10 +8415,10 @@ static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList const char* msg1 = "invalid table name"; int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); - assert(pSqlNode->from->type == SQL_NODE_FROM_TABLELIST); + assert(pSqlNode->from->type == SQL_FROM_NODE_TABLES); for(int32_t j = 0; j < numOfTables; ++j) { - SRelElementPair* item = taosArrayGet(pSqlNode->from->list, j); + SRelElement* item = taosArrayGet(pSqlNode->from->list, j); SStrToken* t = &item->tableName; if (t->type == TK_INTEGER || t->type == TK_FLOAT) { @@ -8446,12 +8446,12 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis int32_t numOfSub = (int32_t) taosArrayGetSize(pSqlNode->from->list); for(int32_t j = 0; j < numOfSub; ++j) { - SRelElementPair* sub = taosArrayGet(pSqlNode->from->list, j); + SRelElement* sub = taosArrayGet(pSqlNode->from->list, j); int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery); for (int32_t i = 0; i < num; ++i) { SSqlNode* p = taosArrayGetP(sub->pSubquery, i); - if (p->from->type == SQL_NODE_FROM_TABLELIST) { + if (p->from->type == SQL_FROM_NODE_TABLES) { int32_t code = getTableNameFromSqlNode(p, tableNameList, msgBuf, pSql); if (code != TSDB_CODE_SUCCESS) { return code; @@ -8520,7 +8520,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // load the table meta in the from clause - if (pSqlNode->from->type == SQL_NODE_FROM_TABLELIST) { + if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { code = getTableNameFromSqlNode(pSqlNode, tableNameList, tscGetErrorMsgPayload(pCmd), pSql); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -8678,7 +8678,7 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod tscAddEmptyMetaInfo(pQueryInfo); } - SRelElementPair *item = taosArrayGet(pSqlNode->from->list, i); + SRelElement *item = taosArrayGet(pSqlNode->from->list, i); SStrToken *oriName = &item->tableName; if (oriName->type == TK_INTEGER || oriName->type == TK_FLOAT) { @@ -8786,7 +8786,7 @@ static STableMeta* extractTempTableMetaFromSubquery(SQueryInfo* pUpstream) { } static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pSql, SQueryInfo* pQueryInfo, char* msgBuf) { - SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, index); + SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, index); // union all is not support currently SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); @@ -8890,7 +8890,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return doLocalQueryProcess(pCmd, pQueryInfo, pSqlNode); } - if (pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) { + if (pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { clearAllTableMetaInfo(pQueryInfo, false, pSql->self); pQueryInfo->numOfTables = 0; @@ -8898,9 +8898,9 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); for (int32_t i = 0; i < numOfSub; ++i) { // check if there is 3 level select - SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, i); + SRelElement* subInfo = taosArrayGet(pSqlNode->from->list, i); SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0); - if (p->from->type == SQL_NODE_FROM_SUBQUERY) { + if (p->from->type == SQL_FROM_NODE_SUBQUERY) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } @@ -9379,8 +9379,8 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { void normalizeSqlNode(SSqlNode* pSqlNode, const char* dbName) { assert(pSqlNode != NULL); - if (pSqlNode->from->type == SQL_NODE_FROM_TABLELIST) { -// SRelElementPair *item = taosArrayGet(pSqlNode->from->list, 0); + if (pSqlNode->from->type == SQL_FROM_NODE_TABLES) { +// SRelElement *item = taosArrayGet(pSqlNode->from->list, 0); // item->TableName.name; } diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 445380f40a..0c60319f39 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -40,8 +40,8 @@ enum SQL_NODE_TYPE { }; enum SQL_NODE_FROM_TYPE { - SQL_NODE_FROM_SUBQUERY = 1, - SQL_NODE_FROM_TABLELIST = 2, + SQL_FROM_NODE_SUBQUERY = 1, + SQL_FROM_NODE_TABLES = 2, }; enum SQL_EXPR_FLAG { @@ -113,18 +113,18 @@ typedef struct SSqlNode { struct tSqlExpr *pHaving; // having clause [optional] } SSqlNode; -typedef struct SRelElementPair { +typedef struct SRelElement { union { SStrToken tableName; SArray *pSubquery; }; SStrToken aliasName; -} SRelElementPair; +} SRelElement; typedef struct SRelationInfo { int32_t type; // nested query|table name list - SArray *list; // SArray + SArray *list; // SArray } SRelationInfo; typedef struct SCreatedTableInfo { diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index c6c5690b2c..cca7bc6148 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -556,11 +556,11 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int SRelationInfo *setTableNameList(SRelationInfo* pRelationInfo, SStrToken *pName, SStrToken* pAlias) { if (pRelationInfo == NULL) { pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); } - pRelationInfo->type = SQL_NODE_FROM_TABLELIST; - SRelElementPair p = {.tableName = *pName}; + pRelationInfo->type = SQL_FROM_NODE_TABLES; + SRelElement p = {.tableName = *pName}; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -576,7 +576,7 @@ void* destroyRelationInfo(SRelationInfo* pRelationInfo) { return NULL; } - if (pRelationInfo->type == SQL_NODE_FROM_TABLELIST) { + if (pRelationInfo->type == SQL_FROM_NODE_TABLES) { taosArrayDestroy(pRelationInfo->list); } else { size_t size = taosArrayGetSize(pRelationInfo->list); @@ -594,12 +594,12 @@ void* destroyRelationInfo(SRelationInfo* pRelationInfo) { SRelationInfo* addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrToken* pAlias) { if (pRelationInfo == NULL) { pRelationInfo = calloc(1, sizeof(SRelationInfo)); - pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElement)); } - pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; + pRelationInfo->type = SQL_FROM_NODE_SUBQUERY; - SRelElementPair p = {.pSubquery = pSub}; + SRelElement p = {.pSubquery = pSub}; if (pAlias != NULL) { p.aliasName = *pAlias; } else { @@ -972,6 +972,7 @@ void SqlInfoDestroy(SSqlInfo *pInfo) { SArray* setSubclause(SArray* pList, void *pSqlNode) { if (pList == NULL) { + pList = taosArrayInit(1, POINTER_BYTES); }