diff --git a/source/libs/scalar/inc/tsclfunc.h b/include/libs/scalar/filter.h
similarity index 54%
rename from source/libs/scalar/inc/tsclfunc.h
rename to include/libs/scalar/filter.h
index 8e03470033..df83a0de35 100644
--- a/source/libs/scalar/inc/tsclfunc.h
+++ b/include/libs/scalar/filter.h
@@ -12,34 +12,25 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
-#ifndef TDENGINE_TSCALARFUNCTION_H
-#define TDENGINE_TSCALARFUNCTION_H
+#ifndef TDENGINE_SCALAR_H
+#define TDENGINE_SCALAR_H
#ifdef __cplusplus
extern "C" {
#endif
#include "function.h"
-#include "scalar.h"
+#include "nodes.h"
-typedef struct SScalarFunctionSupport {
- struct SExprInfo *pExprInfo;
- int32_t numOfCols;
- SColumnInfo *colList;
- void *exprList; // client side used
- int32_t offset;
- char** data;
-} SScalarFunctionSupport;
+typedef struct SFilterInfo SFilterInfo;
-extern struct SScalarFunctionInfo scalarFunc[8];
-
-int32_t evaluateExprNodeTree(tExprNode* pExprs, int32_t numOfRows, SScalarParam* pOutput,
- void* param, char* (*getSourceDataBlock)(void*, const char*, int32_t));
+int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
+int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
#ifdef __cplusplus
}
#endif
-#endif // TDENGINE_TSCALARFUNCTION_H
+#endif // TDENGINE_SCALAR_H
diff --git a/source/libs/scalar/inc/filter.h b/source/libs/scalar/inc/filterInt.h
similarity index 80%
rename from source/libs/scalar/inc/filter.h
rename to source/libs/scalar/inc/filterInt.h
index 27665da0c8..a23e698deb 100644
--- a/source/libs/scalar/inc/filter.h
+++ b/source/libs/scalar/inc/filterInt.h
@@ -13,8 +13,8 @@
* along with this program. If not, see .
*/
-#ifndef TDENGINE_QFILTER_H
-#define TDENGINE_QFILTER_H
+#ifndef TDENGINE_FILTER_INT_H
+#define TDENGINE_FILTER_INT_H
#ifdef __cplusplus
extern "C" {
@@ -85,6 +85,12 @@ enum {
RANGE_TYPE_MR_CTX = 3,
};
+enum {
+ FI_ACTION_NO_NEED = 1,
+ FI_ACTION_CONTINUE,
+ FI_ACTION_STOP,
+};
+
typedef struct OptrStr {
uint16_t optr;
char *str;
@@ -97,17 +103,11 @@ typedef struct SFilterRange {
char eflag;
} SFilterRange;
-typedef struct SFilterColRange {
- uint16_t idx; //column field idx
- bool isNull;
- bool notNull;
- bool isRange;
- SFilterRange ra;
-} SFilterColRange;
-
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
+typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
+typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **);
typedef struct SFilterRangeCompare {
int64_t s;
@@ -155,37 +155,39 @@ typedef struct SFilterField {
} SFilterField;
typedef struct SFilterFields {
- uint16_t size;
- uint16_t num;
+ uint32_t size;
+ uint32_t num;
SFilterField *fields;
} SFilterFields;
typedef struct SFilterFieldId {
uint16_t type;
- uint16_t idx;
+ uint32_t idx;
} SFilterFieldId;
typedef struct SFilterGroup {
- uint16_t unitSize;
- uint16_t unitNum;
- uint16_t *unitIdxs;
+ uint32_t unitSize;
+ uint32_t unitNum;
+ uint32_t *unitIdxs;
uint8_t *unitFlags; // !unit result
} SFilterGroup;
typedef struct SFilterColInfo {
uint8_t type;
int32_t dataType;
+ uint8_t optr; // for equal operation in the relation of RELATION_IN
+ int64_t value; // for equal operation in the relation of RELATION_IN
void *info;
} SFilterColInfo;
typedef struct SFilterGroupCtx {
- uint16_t colNum;
- uint16_t *colIdx;
+ uint32_t colNum;
+ uint32_t *colIdx;
SFilterColInfo *colInfo;
} SFilterGroupCtx;
typedef struct SFilterColCtx {
- uint16_t colIdx;
+ uint32_t colIdx;
void* ctx;
} SFilterColCtx;
@@ -219,16 +221,31 @@ typedef struct SFilterPCtx {
SHashObj *unitHash;
} SFilterPCtx;
+typedef struct SFltTreeStat {
+ int32_t code;
+} SFltTreeStat;
+
+typedef struct SFltScalarCtx {
+ SNode *node;
+} SFltScalarCtx;
+
+typedef struct SFltBuildGroupCtx {
+ SFilterInfo *info;
+ SArray *group;
+ int32_t code;
+} SFltBuildGroupCtx;
+
typedef struct SFilterInfo {
+ bool scalarMode;
+ SFltScalarCtx sclCtx;
uint32_t options;
uint32_t status;
- uint16_t unitSize;
- uint16_t unitNum;
- uint16_t groupNum;
- uint16_t colRangeNum;
+ uint32_t unitSize;
+ uint32_t unitNum;
+ uint32_t groupNum;
+ uint32_t colRangeNum;
SFilterFields fields[FLD_TYPE_MAX];
SFilterGroup *groups;
- uint16_t *cgroups;
SFilterUnit *units;
SFilterComUnit *cunits;
uint8_t *unitRes; // result
@@ -236,16 +253,15 @@ typedef struct SFilterInfo {
SFilterRangeCtx **colRange;
filter_exec_func func;
uint8_t blkFlag;
- uint16_t blkGroupNum;
- uint16_t *blkUnits;
+ uint32_t blkGroupNum;
+ uint32_t *blkUnits;
int8_t *blkUnitRes;
-
+ void *pTable;
+
SFilterPCtx pctx;
} SFilterInfo;
-#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
-
-#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
+#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
@@ -261,7 +277,7 @@ typedef struct SFilterInfo {
#define FILTER_CLR_FLAG(st, f) st &= (~f)
#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
-#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
+#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0)
#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
@@ -271,15 +287,20 @@ typedef struct SFilterInfo {
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
-#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
-#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
-#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
-#define CHK_RETV(c) do { if (c) { return; } } while (0)
-#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
-#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
-#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
-#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
+#define fltFatal(...) qFatal(__VA_ARGS__)
+#define fltError(...) qError(__VA_ARGS__)
+#define fltWarn(...) qWarn(__VA_ARGS__)
+#define fltInfo(...) qInfo(__VA_ARGS__)
+#define fltDebug(...) qDebug(__VA_ARGS__)
+#define fltTrace(...) qTrace(__VA_ARGS__)
+
+
+#define FLT_CHK_JMP(c) do { if (c) { goto _return; } } while (0)
+#define FLT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define FLT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define FLT_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
@@ -290,6 +311,7 @@ typedef struct SFilterInfo {
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
+#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
@@ -302,6 +324,7 @@ typedef struct SFilterInfo {
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
+#define FILTER_UNIT_JSON_VAL_DATA(i, u) FILTER_GET_JSON_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
@@ -316,7 +339,7 @@ typedef struct SFilterInfo {
#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
-#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
+#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint32_t) * n); memcpy(*(dst), src, sizeof(uint32_t) * n);} while (0)
#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
@@ -325,9 +348,10 @@ typedef struct SFilterInfo {
#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
#if 0
-extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
+extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
-extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
+extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp);
+extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
@@ -344,4 +368,4 @@ extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
}
#endif
-#endif // TDENGINE_QFILTER_H
+#endif // TDENGINE_FILTER_INT_H
\ No newline at end of file
diff --git a/source/libs/scalar/inc/tfilter.h b/source/libs/scalar/inc/tfilter.h
deleted file mode 100644
index 27665da0c8..0000000000
--- a/source/libs/scalar/inc/tfilter.h
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc.
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-
-#ifndef TDENGINE_QFILTER_H
-#define TDENGINE_QFILTER_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "thash.h"
-#include "tname.h"
-#include "common.h"
-#include "scalar.h"
-#include "querynodes.h"
-
-#define FILTER_DEFAULT_GROUP_SIZE 4
-#define FILTER_DEFAULT_UNIT_SIZE 4
-#define FILTER_DEFAULT_FIELD_SIZE 4
-#define FILTER_DEFAULT_VALUE_SIZE 4
-#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2
-
-#define FILTER_DUMMY_EMPTY_OPTR 127
-
-#define MAX_NUM_STR_SIZE 40
-
-#define FILTER_RM_UNIT_MIN_ROWS 100
-
-enum {
- FLD_TYPE_COLUMN = 1,
- FLD_TYPE_VALUE = 2,
- FLD_TYPE_MAX = 3,
- FLD_DESC_NO_FREE = 4,
- FLD_DATA_NO_FREE = 8,
- FLD_DATA_IS_HASH = 16,
-};
-
-enum {
- MR_ST_START = 1,
- MR_ST_FIN = 2,
- MR_ST_ALL = 4,
- MR_ST_EMPTY = 8,
-};
-
-enum {
- RANGE_FLG_EXCLUDE = 1,
- RANGE_FLG_INCLUDE = 2,
- RANGE_FLG_NULL = 4,
-};
-
-enum {
- FI_OPTION_NO_REWRITE = 1,
- FI_OPTION_TIMESTAMP = 2,
- FI_OPTION_NEED_UNIQE = 4,
-};
-
-enum {
- FI_STATUS_ALL = 1,
- FI_STATUS_EMPTY = 2,
- FI_STATUS_REWRITE = 4,
- FI_STATUS_CLONED = 8,
-};
-
-enum {
- FI_STATUS_BLK_ALL = 1,
- FI_STATUS_BLK_EMPTY = 2,
- FI_STATUS_BLK_ACTIVE = 4,
-};
-
-enum {
- RANGE_TYPE_UNIT = 1,
- RANGE_TYPE_VAR_HASH = 2,
- RANGE_TYPE_MR_CTX = 3,
-};
-
-typedef struct OptrStr {
- uint16_t optr;
- char *str;
-} OptrStr;
-
-typedef struct SFilterRange {
- int64_t s;
- int64_t e;
- char sflag;
- char eflag;
-} SFilterRange;
-
-typedef struct SFilterColRange {
- uint16_t idx; //column field idx
- bool isNull;
- bool notNull;
- bool isRange;
- SFilterRange ra;
-} SFilterColRange;
-
-typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
-typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
-typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SColumnDataAgg *, int16_t);
-
-typedef struct SFilterRangeCompare {
- int64_t s;
- int64_t e;
- rangeCompFunc func;
-} SFilterRangeCompare;
-
-typedef struct SFilterRangeNode {
- struct SFilterRangeNode* prev;
- struct SFilterRangeNode* next;
- union {
- SFilterRange ra;
- SFilterRangeCompare rc;
- };
-} SFilterRangeNode;
-
-typedef struct SFilterRangeCtx {
- int32_t type;
- int32_t options;
- int8_t status;
- bool isnull;
- bool notnull;
- bool isrange;
- int16_t colId;
- __compar_fn_t pCompareFunc;
- SFilterRangeNode *rf; //freed
- SFilterRangeNode *rs;
-} SFilterRangeCtx ;
-
-typedef struct SFilterVarCtx {
- int32_t type;
- int32_t options;
- int8_t status;
- bool isnull;
- bool notnull;
- bool isrange;
- SHashObj *wild;
- SHashObj *value;
-} SFilterVarCtx;
-
-typedef struct SFilterField {
- uint16_t flag;
- void* desc;
- void* data;
-} SFilterField;
-
-typedef struct SFilterFields {
- uint16_t size;
- uint16_t num;
- SFilterField *fields;
-} SFilterFields;
-
-typedef struct SFilterFieldId {
- uint16_t type;
- uint16_t idx;
-} SFilterFieldId;
-
-typedef struct SFilterGroup {
- uint16_t unitSize;
- uint16_t unitNum;
- uint16_t *unitIdxs;
- uint8_t *unitFlags; // !unit result
-} SFilterGroup;
-
-typedef struct SFilterColInfo {
- uint8_t type;
- int32_t dataType;
- void *info;
-} SFilterColInfo;
-
-typedef struct SFilterGroupCtx {
- uint16_t colNum;
- uint16_t *colIdx;
- SFilterColInfo *colInfo;
-} SFilterGroupCtx;
-
-typedef struct SFilterColCtx {
- uint16_t colIdx;
- void* ctx;
-} SFilterColCtx;
-
-typedef struct SFilterCompare {
- uint8_t type;
- uint8_t optr;
- uint8_t optr2;
-} SFilterCompare;
-
-typedef struct SFilterUnit {
- SFilterCompare compare;
- SFilterFieldId left;
- SFilterFieldId right;
- SFilterFieldId right2;
-} SFilterUnit;
-
-typedef struct SFilterComUnit {
- void *colData;
- void *valData;
- void *valData2;
- uint16_t colId;
- uint16_t dataSize;
- uint8_t dataType;
- uint8_t optr;
- int8_t func;
- int8_t rfunc;
-} SFilterComUnit;
-
-typedef struct SFilterPCtx {
- SHashObj *valHash;
- SHashObj *unitHash;
-} SFilterPCtx;
-
-typedef struct SFilterInfo {
- uint32_t options;
- uint32_t status;
- uint16_t unitSize;
- uint16_t unitNum;
- uint16_t groupNum;
- uint16_t colRangeNum;
- SFilterFields fields[FLD_TYPE_MAX];
- SFilterGroup *groups;
- uint16_t *cgroups;
- SFilterUnit *units;
- SFilterComUnit *cunits;
- uint8_t *unitRes; // result
- uint8_t *unitFlags; // got result
- SFilterRangeCtx **colRange;
- filter_exec_func func;
- uint8_t blkFlag;
- uint16_t blkGroupNum;
- uint16_t *blkUnits;
- int8_t *blkUnitRes;
-
- SFilterPCtx pctx;
-} SFilterInfo;
-
-#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
-
-#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR)
-#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
-
-#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
-
-#define SET_AND_OPTR(ctx, o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { if (!(ctx)->isrange) { (ctx)->notnull = true; } } else if (o != FILTER_DUMMY_EMPTY_OPTR) { (ctx)->isrange = true; (ctx)->notnull = false; } } while (0)
-#define SET_OR_OPTR(ctx,o) do {if (o == TSDB_RELATION_ISNULL) { (ctx)->isnull = true; } else if (o == TSDB_RELATION_NOTNULL) { (ctx)->notnull = true; (ctx)->isrange = false; } else if (o != FILTER_DUMMY_EMPTY_OPTR) { if (!(ctx)->notnull) { (ctx)->isrange = true; } } } while (0)
-#define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true)
-#define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true)))
-
-
-#define FILTER_GET_FLAG(st, f) (st & f)
-#define FILTER_SET_FLAG(st, f) st |= (f)
-#define FILTER_CLR_FLAG(st, f) st &= (~f)
-
-#define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src)
-#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint16_t *)(_t + 1) = idx1; *(uint16_t *)(_t + 3) = idx2; } while (0)
-#define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE))))
-#define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0)
-
-#define RESET_RANGE(ctx, r) do { (r)->next = (ctx)->rf; (ctx)->rf = r; } while (0)
-#define FREE_RANGE(ctx, r) do { if ((r)->prev) { (r)->prev->next = (r)->next; } else { (ctx)->rs = (r)->next;} if ((r)->next) { (r)->next->prev = (r)->prev; } RESET_RANGE(ctx, r); } while (0)
-#define FREE_FROM_RANGE(ctx, r) do { SFilterRangeNode *_r = r; if ((_r)->prev) { (_r)->prev->next = NULL; } else { (ctx)->rs = NULL;} while (_r) {SFilterRangeNode *n = (_r)->next; RESET_RANGE(ctx, _r); _r = n; } } while (0)
-#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
-#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
-
-#define ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
-#define ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
-#define ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
-
-#define CHK_RETV(c) do { if (c) { return; } } while (0)
-#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
-#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
-#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
-#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
-
-#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
-#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
-#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
-#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
-#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId)
-#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
-#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
-#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
-#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
-#define FILTER_GET_TYPE(fl) ((fl) & FLD_TYPE_MAX)
-
-#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
-#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)
-#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
-#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2)
-#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
-#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
-#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
-#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
-#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
-#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
-#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
-#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
-#define FILTER_UNIT_COMP_FUNC(u) ((u)->compare.func)
-
-#define FILTER_UNIT_CLR_F(i) memset((i)->unitFlags, 0, (i)->unitNum * sizeof(*info->unitFlags))
-#define FILTER_UNIT_SET_F(i, idx) (i)->unitFlags[idx] = 1
-#define FILTER_UNIT_GET_F(i, idx) ((i)->unitFlags[idx])
-#define FILTER_UNIT_GET_R(i, idx) ((i)->unitRes[idx])
-#define FILTER_UNIT_SET_R(i, idx, v) (i)->unitRes[idx] = (v)
-
-#define FILTER_PUSH_UNIT(colInfo, u) do { (colInfo).type = RANGE_TYPE_UNIT; (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u);taosArrayPush((SArray *)((colInfo).info), &u);} while (0)
-#define FILTER_PUSH_VAR_HASH(colInfo, ha) do { (colInfo).type = RANGE_TYPE_VAR_HASH; (colInfo).info = ha;} while (0)
-#define FILTER_PUSH_CTX(colInfo, ctx) do { (colInfo).type = RANGE_TYPE_MR_CTX; (colInfo).info = ctx;} while (0)
-
-#define FILTER_COPY_IDX(dst, src, n) do { *(dst) = malloc(sizeof(uint16_t) * n); memcpy(*(dst), src, sizeof(uint16_t) * n);} while (0)
-
-#define FILTER_ADD_CTX_TO_GRES(gres, idx, ctx) do { if ((gres)->colCtxs == NULL) { (gres)->colCtxs = taosArrayInit(gres->colNum, sizeof(SFilterColCtx)); } SFilterColCtx cCtx = {idx, ctx}; taosArrayPush((gres)->colCtxs, &cCtx); } while (0)
-
-
-#define FILTER_ALL_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_ALL)
-#define FILTER_EMPTY_RES(i) FILTER_GET_FLAG((i)->status, FI_STATUS_EMPTY)
-
-#if 0
-extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
-extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
-extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
-extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
-extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
-extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
-extern void filterFreeInfo(SFilterInfo *info);
-extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
-#else
-//REMOVE THESE!!!!!!!!!!!!!!!!!!!!
-#include "function.h"
-#endif
-extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right);
-extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // TDENGINE_QFILTER_H
diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c
index b680ac6778..cf84e45d4c 100644
--- a/source/libs/scalar/src/filter.c
+++ b/source/libs/scalar/src/filter.c
@@ -17,7 +17,7 @@
#include "thash.h"
//#include "queryLog.h"
#include "tcompare.h"
-#include "filter.h"
+#include "filterInt.h"
OptrStr gOptrStr[] = {
{TSDB_RELATION_INVALID, "invalid"},
@@ -28,21 +28,22 @@ OptrStr gOptrStr[] = {
{TSDB_RELATION_GREATER_EQUAL, ">="},
{TSDB_RELATION_NOT_EQUAL, "!="},
{TSDB_RELATION_LIKE, "like"},
- {TSDB_RELATION_MATCH, "match"},
- {TSDB_RELATION_MATCH, "nmatch"},
{TSDB_RELATION_ISNULL, "is null"},
{TSDB_RELATION_NOTNULL, "not null"},
{TSDB_RELATION_IN, "in"},
{TSDB_RELATION_AND, "and"},
{TSDB_RELATION_OR, "or"},
- {TSDB_RELATION_NOT, "not"}
+ {TSDB_RELATION_NOT, "not"},
+ {TSDB_RELATION_MATCH, "match"},
+ {TSDB_RELATION_NMATCH, "nmatch"},
+ {TSDB_RELATION_CONTAINS, "contains"},
};
static FORCE_INLINE int32_t filterFieldColDescCompare(const void *desc1, const void *desc2) {
const SSchema *sch1 = desc1;
const SSchema *sch2 = desc2;
- return sch1->colId != sch2->colId;
+ return !(strcmp(sch1->name, sch2->name) == 0 && sch1->colId == sch2->colId);
}
static FORCE_INLINE int32_t filterFieldValDescCompare(const void *desc1, const void *desc2) {
@@ -60,16 +61,24 @@ filter_desc_compare_func gDescCompare [FLD_TYPE_MAX] = {
};
bool filterRangeCompGi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
- return cfunc(maxv, minr) >= 0;
+ int32_t result = cfunc(maxv, minr);
+ if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+ return result >= 0;
}
bool filterRangeCompGe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
- return cfunc(maxv, minr) > 0;
+ int32_t result = cfunc(maxv, minr);
+ if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+ return result > 0;
}
bool filterRangeCompLi (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
- return cfunc(minv, maxr) <= 0;
+ int32_t result = cfunc(minv, maxr);
+ if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+ return result <= 0;
}
bool filterRangeCompLe (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
- return cfunc(minv, maxr) < 0;
+ int32_t result = cfunc(minv, maxr);
+ if (result == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+ return result < 0;
}
bool filterRangeCompii (const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) {
return cfunc(maxv, minr) >= 0 && cfunc(minv, maxr) <= 0;
@@ -291,7 +300,7 @@ int32_t filterInitUnitsFields(SFilterInfo *info) {
info->fields[FLD_TYPE_COLUMN].num = 0;
info->fields[FLD_TYPE_COLUMN].size = FILTER_DEFAULT_FIELD_SIZE;
- info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, COL_FIELD_SIZE);
+ info->fields[FLD_TYPE_COLUMN].fields = calloc(info->fields[FLD_TYPE_COLUMN].size, sizeof(SFilterField));
info->fields[FLD_TYPE_VALUE].num = 0;
info->fields[FLD_TYPE_VALUE].size = FILTER_DEFAULT_FIELD_SIZE;
info->fields[FLD_TYPE_VALUE].fields = calloc(info->fields[FLD_TYPE_VALUE].size, sizeof(SFilterField));
@@ -318,7 +327,7 @@ static FORCE_INLINE SFilterRangeNode* filterNewRange(SFilterRangeCtx *ctx, SFilt
void* filterInitRangeCtx(int32_t type, int32_t options) {
if (type > TSDB_DATA_TYPE_UBIGINT || type < TSDB_DATA_TYPE_BOOL || type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
- //qError("not supported range type:%d", type);
+ qError("not supported range type:%d", type);
return NULL;
}
@@ -685,7 +694,7 @@ int32_t filterGetRangeRes(void* h, SFilterRange *ra) {
}
if (num == 0) {
- //qError("no range result");
+ qError("no range result");
return TSDB_CODE_QRY_APP_ERROR;
}
@@ -775,8 +784,15 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
int32_t leftSize = (int32_t)taosArrayGetSize(left);
int32_t rightSize = (int32_t)taosArrayGetSize(right);
-// CHK_LRET(taosArrayGetSize(left) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group");
-// CHK_LRET(taosArrayGetSize(right) <= 0, TSDB_CODE_QRY_APP_ERROR, "empty group");
+ if (taosArrayGetSize(left) <= 0) {
+ fltDebug("empty group");
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
+
+ if (taosArrayGetSize(right) <= 0) {
+ fltDebug("empty group");
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
for (int32_t l = 0; l < leftSize; ++l) {
SFilterGroup *gp1 = taosArrayGet(left, l);
@@ -793,7 +809,7 @@ int32_t filterDetachCnfGroups(SArray* group, SArray* left, SArray* right) {
}
int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
- for (uint16_t i = 0; i < fields->num; ++i) {
+ for (uint32_t i = 0; i < fields->num; ++i) {
if (0 == gDescCompare[type](fields->fields[i].desc, v)) {
return i;
}
@@ -806,7 +822,7 @@ int32_t filterGetFiledByDesc(SFilterFields* fields, int32_t type, void *v) {
int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) {
if (type == FLD_TYPE_VALUE) {
if (info->pctx.valHash == false) {
- //qError("value hash is empty");
+ qError("value hash is empty");
return -1;
}
@@ -819,10 +835,11 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d
return -1;
}
-
+// In the params, we should use void *data instead of void **data, there is no need to use tfree(*data) to set *data = 0
+// Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better.
int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, bool freeIfExists) {
int32_t idx = -1;
- uint16_t *num;
+ uint32_t *num;
num = &info->fields[type].num;
@@ -885,8 +902,14 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte
int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldId *fid) {
-// CHK_LRET(node == NULL, TSDB_CODE_QRY_APP_ERROR, "empty node");
-// CHK_RET(node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE, TSDB_CODE_QRY_APP_ERROR);
+ if (node == NULL) {
+ fltError("empty node");
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
+
+ if (node->nodeType != TEXPR_BINARYEXPR_NODE && node->nodeType != TEXPR_VALUE_NODE) {
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
int32_t type;
void *v;
@@ -906,7 +929,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, tExprNode *node, SFilterFieldI
return TSDB_CODE_SUCCESS;
}
-int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint16_t *uidx) {
+int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) {
if (FILTER_GET_FLAG(info->options, FI_OPTION_NEED_UNIQE)) {
if (info->pctx.unitHash == NULL) {
info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false);
@@ -915,14 +938,14 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1);
void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v));
if (hu) {
- *uidx = *(uint16_t *)hu;
+ *uidx = *(uint32_t *)hu;
return TSDB_CODE_SUCCESS;
}
}
}
if (info->unitNum >= info->unitSize) {
- uint16_t psize = info->unitSize;
+ uint32_t psize = info->unitSize;
info->unitSize += FILTER_DEFAULT_UNIT_SIZE;
info->units = realloc(info->units, info->unitSize * sizeof(SFilterUnit));
memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE);
@@ -940,7 +963,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u);
assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE));
} else {
- assert(optr == TSDB_RELATION_ISNULL || optr == TSDB_RELATION_NOTNULL || optr == FILTER_DUMMY_EMPTY_OPTR);
+ if(optr != TSDB_RELATION_ISNULL && optr != TSDB_RELATION_NOTNULL && optr != FILTER_DUMMY_EMPTY_OPTR){
+ return -1;
+ }
}
SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, u);
@@ -963,7 +988,7 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
-int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
+int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) {
if (group->unitNum >= group->unitSize) {
group->unitSize += FILTER_DEFAULT_UNIT_SIZE;
group->unitIdxs = realloc(group->unitIdxs, group->unitSize * sizeof(*group->unitIdxs));
@@ -974,12 +999,14 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) {
return TSDB_CODE_SUCCESS;
}
-int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType) {
+int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint32_t tType, bool tolower) {
SBufferReader br = tbufInitReader(buf, len, false);
uint32_t sType = tbufReadUint32(&br);
SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(tType), true, false);
int32_t code = 0;
-
+
+ taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(tType));
+
int dummy = -1;
SVariant tmpVar = {0};
size_t t = 0;
@@ -1065,47 +1092,53 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
switch (tType) {
case TSDB_DATA_TYPE_BOOL:
+ if (sType != TSDB_DATA_TYPE_BOOL && !IS_SIGNED_NUMERIC_TYPE(sType)) {
+ goto _return;
+ }
+ if (tmpVar.i64 > 1 ||tmpVar.i64 < 0) {
+ goto _return;
+ }
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_TINYINT: {
-// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-// if (converted) {
-// taosVariantDestroy(&tmpVar);
-// memset(&tmpVar, 0, sizeof(tmpVar));
-// continue;
-// }
-//
-// goto _return;
-// }
+ if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
+ if (converted) {
+ tVariantDestroy(&tmpVar);
+ memset(&tmpVar, 0, sizeof(tmpVar));
+ continue;
+ }
+
+ goto _return;
+ }
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_SMALLINT: {
-// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-// if (converted) {
-// taosVariantDestroy(&tmpVar);
-// memset(&tmpVar, 0, sizeof(tmpVar));
-// continue;
-// }
-//
-// goto _return;
-// }
+ if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
+ if (converted) {
+ tVariantDestroy(&tmpVar);
+ memset(&tmpVar, 0, sizeof(tmpVar));
+ continue;
+ }
+
+ goto _return;
+ }
pvar = &val;
t = sizeof(val);
break;
}
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_INT: {
-// if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-// if (converted) {
-// taosVariantDestroy(&tmpVar);
-// memset(&tmpVar, 0, sizeof(tmpVar));
-// continue;
-// }
-//
-// goto _return;
-// }
+ if (tVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
+ if (converted) {
+ tVariantDestroy(&tmpVar);
+ memset(&tmpVar, 0, sizeof(tmpVar));
+ continue;
+ }
+
+ goto _return;
+ }
pvar = &val;
t = sizeof(val);
break;
@@ -1129,15 +1162,15 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
break;
}
case TSDB_DATA_TYPE_FLOAT: {
-// if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
-// if (converted) {
-// taosVariantDestroy(&tmpVar);
-// memset(&tmpVar, 0, sizeof(tmpVar));
-// continue;
-// }
+ if (taosVariantDumpEx(&tmpVar, (char *)&val, tType, false, &converted, &extInfo)) {
+ if (converted) {
+ tVariantDestroy(&tmpVar);
+ memset(&tmpVar, 0, sizeof(tmpVar));
+ continue;
+ }
-// goto _return;
-// }
+ goto _return;
+ }
pvar = &val;
t = sizeof(val);
break;
@@ -1148,6 +1181,10 @@ int32_t filterConvertSetFromBinary(void **q, const char *buf, int32_t len, uint3
}
t = varDataLen(tmp);
pvar = varDataVal(tmp);
+
+ if (tolower) {
+ strntolower_s(pvar, pvar, (int32_t)t);
+ }
break;
}
case TSDB_DATA_TYPE_NCHAR: {
@@ -1178,22 +1215,51 @@ _return:
return code;
}
+static int32_t filterDealJson(SFilterInfo *info, tExprNode* tree, tExprNode** pLeft) {
+ if((*pLeft)->nodeType == TSQL_NODE_EXPR && (*pLeft)->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation
+ assert(info->pTable != NULL);
+ SSchema* schema = (*pLeft)->_node.pLeft->pSchema;
+ if((*pLeft)->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+ char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
+ jsonKeyMd5((*pLeft)->_node.pRight->pVal->pz, (*pLeft)->_node.pRight->pVal->nLen, keyMd5);
+ memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
+ (*pLeft) = (*pLeft)->_node.pLeft; // -> operation use left as input
+ }else if(((*pLeft)->pSchema->type == TSDB_DATA_TYPE_JSON) &&
+ (tree->_node.optr == TSDB_RELATION_ISNULL || tree->_node.optr == TSDB_RELATION_NOTNULL)){
+ SSchema* schema = (*pLeft)->pSchema;
+ char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
+ uint32_t nullData = TSDB_DATA_JSON_NULL;
+ jsonKeyMd5(&nullData, INT_BYTES, keyMd5);
+ memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
+ }else if(tree->_node.optr == TSDB_RELATION_CONTAINS){
+ SSchema* schema = (*pLeft)->pSchema;
+ if(tree->_node.pRight->pVal->nLen > TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+ char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
+ jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5);
+ memcpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
+ }
+ return TSDB_CODE_SUCCESS;
+}
-int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) {
+int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
+ tExprNode* pLeft = tree->_node.pLeft;
+ int32_t ret = TSDB_CODE_SUCCESS;
+ if((ret = filterDealJson(info, tree, &pLeft)) != TSDB_CODE_SUCCESS) return ret;
SFilterFieldId left = {0}, right = {0};
-
- filterAddFieldFromNode(info, tree->_node.pLeft, &left);
-
- SVariant* var = tree->_node.pRight->pVal;
- int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
- size_t len = 0;
- uint16_t uidx = 0;
+ filterAddFieldFromNode(info, pLeft, &left);
+ uint8_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left));
+ int32_t len = 0;
+ uint32_t uidx = 0;
if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) {
void *data = NULL;
- filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type);
-// CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param");
+ SVariant* var = tree->_node.pRight->pVal;
+ filterConvertSetFromBinary((void **)&data, var->pz, var->nLen, type, false);
+ if (data == NULL) {
+ fltError("failed to convert in param");
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
if (taosHashGetSize((SHashObj *)data) <= 0) {
filterAddUnit(info, FILTER_DUMMY_EMPTY_OPTR, &left, NULL, &uidx);
@@ -1209,23 +1275,13 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
void *p = taosHashIterate((SHashObj *)data, NULL);
while(p) {
- void* key = NULL;
- len = 0;
-
- taosHashGetKey(p, &key, &len);
+ void *key = taosHashGetDataKey((SHashObj *)data, p);
void *fdata = NULL;
-
- if (IS_VAR_DATA_TYPE(type)) {
- fdata = malloc(len + VARSTR_HEADER_SIZE);
- varDataLen(fdata) = len;
- memcpy(varDataVal(fdata), key, len);
- len += VARSTR_HEADER_SIZE;
- } else {
- fdata = malloc(sizeof(int64_t));
- SIMPLE_COPY_VALUES(fdata, key);
- len = tDataTypes[type].bytes;
- }
-
+
+ fdata = malloc(sizeof(int64_t));
+ SIMPLE_COPY_VALUES(fdata, key);
+ len = tDataTypes[type].bytes;
+
filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true);
filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right, &uidx);
@@ -1240,9 +1296,10 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
taosHashCleanup(data);
} else {
- filterAddFieldFromNode(info, tree->_node.pRight, &right);
+ filterAddFieldFromNode(info, tree->_node.pRight, &right);
- filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
+ ret = filterAddUnit(info, tree->_node.optr, &left, &right, &uidx);
+ CHK_LRET(ret != TSDB_CODE_SUCCESS, TSDB_CODE_QRY_APP_ERROR, "invalid where condition");
SFilterGroup fgroup = {0};
filterAddUnitToGroup(&fgroup, uidx);
@@ -1254,7 +1311,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
}
-int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint16_t *uidx) {
+int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u, uint32_t *uidx) {
SFilterFieldId left, right, *pright = &right;
int32_t type = FILTER_UNIT_DATA_TYPE(u);
uint16_t flag = FLD_DESC_NO_FREE;
@@ -1267,7 +1324,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
void *data = FILTER_UNIT_VAL_DATA(src, u);
if (IS_VAR_DATA_TYPE(type)) {
if (FILTER_UNIT_OPTR(u) == TSDB_RELATION_IN) {
- filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, 0, false);
+ filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right.
t = FILTER_GET_FIELD(dst, right);
@@ -1289,7 +1346,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u
return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, pright, uidx);
}
-int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint16_t uidx) {
+int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *right, uint32_t uidx) {
SFilterUnit *u = &info->units[uidx];
u->compare.optr2 = optr;
@@ -1299,9 +1356,9 @@ int32_t filterAddUnitRight(SFilterInfo *info, uint8_t optr, SFilterFieldId *righ
}
-int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint16_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
+int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRangeCtx *ctx, uint32_t cidx, SFilterGroup *g, int32_t optr, SArray *res) {
SFilterFieldId left, right, right2;
- uint16_t uidx = 0;
+ uint32_t uidx = 0;
SFilterField *col = FILTER_GET_COL_FIELD(src, cidx);
@@ -1479,61 +1536,80 @@ static void filterFreeGroup(void *pItem) {
}
-int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) {
+EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
int32_t code = TSDB_CODE_SUCCESS;
- SArray* leftGroup = NULL;
- SArray* rightGroup = NULL;
-
- if (tree->nodeType != TEXPR_BINARYEXPR_NODE) {
- //qError("invalid nodeType:%d", tree->nodeType);
- return TSDB_CODE_QRY_APP_ERROR;
+ SArray* preGroup = NULL;
+ SArray* newGroup = NULL;
+ SArray* resGroup = NULL;
+ ENodeType nType = nodeType(nType);
+ SFltBuildGroupCtx *ctx = (SFltBuildGroupCtx *)pContext;
+
+ if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) {
+ SLogicConditionNode *node = (SLogicConditionNode *)pNode;
+ if (LOGIC_COND_TYPE_AND == node->condType) {
+ SListCell *cell = node->pParameterList->pHead;
+ for (int32_t i = 0; i < node->pParameterList->length; ++i) {
+ newGroup = taosArrayInit(4, sizeof(SFilterGroup));
+ resGroup = taosArrayInit(4, sizeof(SFilterGroup));
+
+ SFltBuildGroupCtx tctx = {.info = ctx->info, .group = newGroup};
+ nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)&tctx);
+ FLT_ERR_JRET(tctx.code);
+
+ FLT_ERR_JRET(filterDetachCnfGroups(resGroup, preGroup, newGroup));
+
+ taosArrayDestroyEx(newGroup, filterFreeGroup);
+ taosArrayDestroyEx(preGroup, filterFreeGroup);
+
+ preGroup = resGroup;
+ }
+
+ taosArrayAddAll(ctx->group, resGroup);
+
+ taosArrayDestroyEx(newGroup, filterFreeGroup);
+ taosArrayDestroyEx(preGroup, filterFreeGroup);
+ taosArrayDestroyEx(resGroup, filterFreeGroup);
+
+ return DEAL_RES_IGNORE_CHILD;
+ }
+
+ if (LOGIC_COND_TYPE_OR == node->condType) {
+ SListCell *cell = node->pParameterList->pHead;
+ for (int32_t i = 0; i < node->pParameterList->length; ++i) {
+ nodesWalkNode(cell->pNode, fltTreeToGroup, (void *)pContext);
+ FLT_ERR_JRET(ctx.code);
+ }
+
+ return DEAL_RES_IGNORE_CHILD;
+ }
+
+ fltError("invalid condition type, type:%d", node->condType);
+
+ return DEAL_RES_ERROR;
}
-
- if (tree->_node.optr == TSDB_RELATION_AND) {
- leftGroup = taosArrayInit(4, sizeof(SFilterGroup));
- rightGroup = taosArrayInit(4, sizeof(SFilterGroup));
- ERR_JRET(filterTreeToGroup(tree->_node.pLeft, info, leftGroup));
- ERR_JRET(filterTreeToGroup(tree->_node.pRight, info, rightGroup));
- ERR_JRET(filterDetachCnfGroups(group, leftGroup, rightGroup));
-
- taosArrayDestroyEx(leftGroup, filterFreeGroup);
- taosArrayDestroyEx(rightGroup, filterFreeGroup);
+ if (QUERY_NODE_OPERATOR == nType) {
+ FLT_ERR_JRET(fltAddGroupUnitFromNode(ctx->info, pNode, ctx->group));
- return TSDB_CODE_SUCCESS;
+ return DEAL_RES_CONTINUE;
}
- if (tree->_node.optr == TSDB_RELATION_OR) {
- ERR_RET(filterTreeToGroup(tree->_node.pLeft, info, group));
- ERR_RET(filterTreeToGroup(tree->_node.pRight, info, group));
-
- return TSDB_CODE_SUCCESS;
- }
-
- code = filterAddGroupUnitFromNode(info, tree, group);
+ sclError("invalid node type for filter, type:%d", nodeType(pNode));
+ code = TSDB_CODE_QRY_INVALID_INPUT;
_return:
- taosArrayDestroyEx(leftGroup, filterFreeGroup);
- taosArrayDestroyEx(rightGroup, filterFreeGroup);
-
- return code;
+ taosArrayDestroyEx(newGroup, filterFreeGroup);
+ taosArrayDestroyEx(preGroup, filterFreeGroup);
+ taosArrayDestroyEx(resGroup, filterFreeGroup);
+
+ ctx->code = code;
+
+ return DEAL_RES_ERROR;
}
-#if 0
-int32_t filterInitUnitFunc(SFilterInfo *info) {
- for (uint16_t i = 0; i < info->unitNum; ++i) {
- SFilterUnit* unit = &info->units[i];
-
- info->cunits[i].func = getComparFunc(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
- }
-
- return TSDB_CODE_SUCCESS;
-}
-#endif
-
-int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
+int32_t fltConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t n = 0;
switch (type) {
@@ -1611,34 +1687,37 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) {
if (qDebugFlag & DEBUG_DEBUG) {
-// CHK_LRETV(info == NULL, "%s - FilterInfo: EMPTY", msg);
+ if (info == NULL) {
+ fltDebug("%s - FilterInfo: EMPTY", msg);
+ return;
+ }
if (options == 0) {
-// //qDebug("%s - FilterInfo:", msg);
-// //qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
- for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+ qDebug("%s - FilterInfo:", msg);
+ qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
+ for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema *sch = field->desc;
-// //qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name);
+ qDebug("COL%d => [%d][%s]", i, sch->colId, sch->name);
}
- //qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
- for (uint16_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
+ qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num);
+ for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i];
if (field->desc) {
SVariant *var = field->desc;
if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) {
- //qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1));
+ qDebug("VAL%d => [type:TS][val:[%" PRIi64"] - [%" PRId64 "]]", i, *(int64_t *)field->data, *(((int64_t *)field->data) + 1));
} else {
- //qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO
+ qDebug("VAL%d => [type:%d][val:%" PRIx64"]", i, var->nType, var->i64); //TODO
}
} else if (field->data) {
- //qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
+ qDebug("VAL%d => [type:NIL][val:NIL]", i); //TODO
}
}
- //qDebug("UNIT Num:%u", info->unitNum);
- for (uint16_t i = 0; i < info->unitNum; ++i) {
+ qDebug("UNIT Num:%u", info->unitNum);
+ for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
int32_t type = FILTER_UNIT_DATA_TYPE(unit);
int32_t len = 0;
@@ -1647,7 +1726,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SSchema *sch = left->desc;
- len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
+ if (unit->compare.optr >= TSDB_RELATION_INVALID && unit->compare.optr <= TSDB_RELATION_CONTAINS){
+ len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str);
+ }
if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit);
@@ -1656,7 +1737,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE;
}
- converToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
+ if (data) fltConverToStr(str + len, type, data, tlen > 32 ? 32 : tlen, &tlen);
} else {
strcat(str, "NULL");
}
@@ -1664,7 +1745,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
if (unit->compare.optr2) {
strcat(str, " && ");
- sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str);
+ if (unit->compare.optr2 >= TSDB_RELATION_INVALID && unit->compare.optr2 <= TSDB_RELATION_CONTAINS){
+ sprintf(str + strlen(str), "[%d][%s] %s [", sch->colId, sch->name, gOptrStr[unit->compare.optr2].str);
+ }
if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) {
SFilterField *right = FILTER_UNIT_RIGHT2_FIELD(info, unit);
@@ -1673,23 +1756,23 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
tlen = varDataLen(data);
data += VARSTR_HEADER_SIZE;
}
- converToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
+ fltConverToStr(str + strlen(str), type, data, tlen > 32 ? 32 : tlen, &tlen);
} else {
strcat(str, "NULL");
}
strcat(str, "]");
}
- //qDebug("%s", str); //TODO
+ qDebug("%s", str); //TODO
}
- //qDebug("GROUP Num:%u", info->groupNum);
- for (uint16_t i = 0; i < info->groupNum; ++i) {
+ qDebug("GROUP Num:%u", info->groupNum);
+ for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup *group = &info->groups[i];
- //qDebug("Group%d : unit num[%u]", i, group->unitNum);
+ qDebug("Group%d : unit num[%u]", i, group->unitNum);
- for (uint16_t u = 0; u < group->unitNum; ++u) {
- //qDebug("unit id:%u", group->unitIdxs[u]);
+ for (uint32_t u = 0; u < group->unitNum; ++u) {
+ qDebug("unit id:%u", group->unitIdxs[u]);
}
}
@@ -1697,12 +1780,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
}
if (options == 1) {
- //qDebug("%s - RANGE info:", msg);
+ qDebug("%s - RANGE info:", msg);
- //qDebug("RANGE Num:%u", info->colRangeNum);
- for (uint16_t i = 0; i < info->colRangeNum; ++i) {
+ qDebug("RANGE Num:%u", info->colRangeNum);
+ for (uint32_t i = 0; i < info->colRangeNum; ++i) {
SFilterRangeCtx *ctx = info->colRange[i];
- //qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
+ qDebug("Column ID[%d] RANGE: isnull[%d],notnull[%d],range[%d]", ctx->colId, ctx->isnull, ctx->notnull, ctx->isrange);
if (ctx->isrange) {
SFilterRangeNode *r = ctx->rs;
while (r) {
@@ -1712,7 +1795,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
strcat(str,"(NULL)");
} else {
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
- converToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
+ fltConverToStr(str + strlen(str), ctx->type, &r->ra.s, tlen > 32 ? 32 : tlen, &tlen);
FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
}
strcat(str, " - ");
@@ -1720,10 +1803,10 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
strcat(str, "(NULL)");
} else {
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,"(") : strcat(str,"[");
- converToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
+ fltConverToStr(str + strlen(str), ctx->type, &r->ra.e, tlen > 32 ? 32 : tlen, &tlen);
FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? strcat(str,")") : strcat(str,"]");
}
- //qDebug("range: %s", str);
+ qDebug("range: %s", str);
r = r->next;
}
@@ -1733,25 +1816,25 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
return;
}
- //qDebug("%s - Block Filter info:", msg);
+ qDebug("%s - Block Filter info:", msg);
if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ALL)) {
- //qDebug("Flag:%s", "ALL");
+ qDebug("Flag:%s", "ALL");
return;
} else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_EMPTY)) {
- //qDebug("Flag:%s", "EMPTY");
+ qDebug("Flag:%s", "EMPTY");
return;
} else if (FILTER_GET_FLAG(info->blkFlag, FI_STATUS_BLK_ACTIVE)){
- //qDebug("Flag:%s", "ACTIVE");
+ qDebug("Flag:%s", "ACTIVE");
}
- //qDebug("GroupNum:%d", info->blkGroupNum);
- uint16_t *unitIdx = info->blkUnits;
- for (uint16_t i = 0; i < info->blkGroupNum; ++i) {
- //qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
- uint16_t unitNum = *(unitIdx++);
- for (uint16_t m = 0; m < unitNum; ++m) {
- //qDebug("uidx[%d]", *(unitIdx++));
+ qDebug("GroupNum:%d", info->blkGroupNum);
+ uint32_t *unitIdx = info->blkUnits;
+ for (uint32_t i = 0; i < info->blkGroupNum; ++i) {
+ qDebug("Group[%d] UnitNum: %d:", i, *unitIdx);
+ uint32_t unitNum = *(unitIdx++);
+ for (uint32_t m = 0; m < unitNum; ++m) {
+ qDebug("uidx[%d]", *(unitIdx++));
}
}
}
@@ -1837,21 +1920,23 @@ void filterFreePCtx(SFilterPCtx *pctx) {
}
void filterFreeInfo(SFilterInfo *info) {
- CHK_RETV(info == NULL);
+ if (info == NULL) {
+ return;
+ }
tfree(info->cunits);
tfree(info->blkUnitRes);
tfree(info->blkUnits);
for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) {
- for (uint16_t f = 0; f < info->fields[i].num; ++f) {
+ for (uint32_t f = 0; f < info->fields[i].num; ++f) {
filterFreeField(&info->fields[i].fields[f], i);
}
tfree(info->fields[i].fields);
}
- for (int32_t i = 0; i < info->groupNum; ++i) {
+ for (uint32_t i = 0; i < info->groupNum; ++i) {
filterFreeGroup(&info->groups[i]);
}
@@ -1863,7 +1948,7 @@ void filterFreeInfo(SFilterInfo *info) {
tfree(info->unitFlags);
- for (uint16_t i = 0; i < info->colRangeNum; ++i) {
+ for (uint32_t i = 0; i < info->colRangeNum; ++i) {
filterFreeRangeCtx(info->colRange[i]);
}
@@ -1901,7 +1986,7 @@ int32_t filterHandleValueExtInfo(SFilterUnit* unit, char extInfo) {
int32_t filterInitValFieldData(SFilterInfo *info) {
- for (uint16_t i = 0; i < info->unitNum; ++i) {
+ for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit* unit = &info->units[i];
if (unit->right.type != FLD_TYPE_VALUE) {
assert(unit->compare.optr == TSDB_RELATION_ISNULL || unit->compare.optr == TSDB_RELATION_NOTNULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR);
@@ -1923,8 +2008,11 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
}
if (unit->compare.optr == TSDB_RELATION_IN) {
- filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type);
-// CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param");
+ filterConvertSetFromBinary((void **)&fi->data, var->pz, var->nLen, type, false);
+ if (fi->data == NULL) {
+ fltError("failed to convert in param");
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH);
@@ -1937,7 +2025,7 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} else if (type == TSDB_DATA_TYPE_NCHAR) {
size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
- } else {
+ } else if (type != TSDB_DATA_TYPE_JSON){
if (var->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { //TIME RANGE
fi->data = calloc(var->nLen, tDataTypes[type].bytes);
for (int32_t a = 0; a < var->nLen; ++a) {
@@ -1949,19 +2037,47 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
} else {
fi->data = calloc(1, sizeof(int64_t));
}
+ } else{ // type == TSDB_DATA_TYPE_JSON
+ // fi->data = null; use fi->desc as data, because json value is variable, so use tVariant (fi->desc)
+ }
+
+ if(type != TSDB_DATA_TYPE_JSON){
+ bool converted = false;
+ char extInfo = 0;
+ if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
+ if (converted) {
+ filterHandleValueExtInfo(unit, extInfo);
+
+ continue;
+ }
+ qError("dump value to type[%d] failed", type);
+ return TSDB_CODE_TSC_INVALID_OPERATION;
+ }
+ }
+
+ // match/nmatch for nchar type need convert from ucs4 to mbs
+ if(type == TSDB_DATA_TYPE_NCHAR &&
+ (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
+ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
+ int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
+ if (len < 0){
+ qError("filterInitValFieldData taosUcs4ToMbs error 1");
+ return TSDB_CODE_FAILED;
+ }
+ varDataSetLen(newValData, len);
+ varDataCopy(fi->data, newValData);
+ }else if(type == TSDB_DATA_TYPE_JSON &&
+ (unit->compare.optr == TSDB_RELATION_MATCH || unit->compare.optr == TSDB_RELATION_NMATCH)){
+ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
+ int32_t len = taosUcs4ToMbs(((tVariant*)(fi->desc))->pz, ((tVariant*)(fi->desc))->nLen, newValData);
+ if (len < 0){
+ qError("filterInitValFieldData taosUcs4ToMbs error 2");
+ return TSDB_CODE_FAILED;
+ }
+ memcpy(((tVariant*)(fi->desc))->pz, newValData, len);
+ ((tVariant*)(fi->desc))->nLen = len;
}
- bool converted = false;
- char extInfo = 0;
-// if (tVariantDumpEx(var, (char*)fi->data, type, true, &converted, &extInfo)) {
-// if (converted) {
-// filterHandleValueExtInfo(unit, extInfo);
-//
-// continue;
-// }
-// //qError("dump value to type[%d] failed", type);
-// return TSDB_CODE_TSC_INVALID_OPERATION;
-// }
}
return TSDB_CODE_SUCCESS;
@@ -1971,6 +2087,8 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right) {
int32_t ret = func(left, right);
+ if(ret == TSDB_DATA_JSON_CAN_NOT_COMPARE) return false;
+
switch (optr) {
case TSDB_RELATION_EQUAL: {
return ret == 0;
@@ -1996,6 +2114,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
case TSDB_RELATION_MATCH: {
return ret == 0;
}
+ case TSDB_RELATION_CONTAINS: {
+ return ret == 0;
+ }
case TSDB_RELATION_NMATCH: {
return ret == 0;
}
@@ -2092,7 +2213,7 @@ _return:
}
-int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colIdx, bool *empty) {
+int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint32_t colIdx, bool *empty) {
SArray* colArray = (SArray *)gRes->colInfo[colIdx].info;
int32_t size = (int32_t)taosArrayGetSize(colArray);
int32_t type = gRes->colInfo[colIdx].dataType;
@@ -2109,6 +2230,10 @@ int32_t filterMergeUnits(SFilterInfo *info, SFilterGroupCtx* gRes, uint16_t colI
filterAddUnitRange(info, u, ctx, TSDB_RELATION_AND);
CHK_JMP(MR_EMPTY_RES(ctx));
}
+ if(FILTER_UNIT_OPTR(u) == TSDB_RELATION_EQUAL && !FILTER_NO_MERGE_DATA_TYPE(FILTER_UNIT_DATA_TYPE(u))){
+ gRes->colInfo[colIdx].optr = TSDB_RELATION_EQUAL;
+ SIMPLE_COPY_VALUES(&gRes->colInfo[colIdx].value, FILTER_UNIT_VAL_DATA(info, u));
+ }
}
taosArrayDestroy(colArray);
@@ -2129,11 +2254,11 @@ _return:
int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t* gResNum) {
bool empty = false;
- uint16_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint16_t));
- uint16_t colIdxi = 0;
- uint16_t gResIdx = 0;
+ uint32_t *colIdx = malloc(info->fields[FLD_TYPE_COLUMN].num * sizeof(uint32_t));
+ uint32_t colIdxi = 0;
+ uint32_t gResIdx = 0;
- for (uint16_t i = 0; i < info->groupNum; ++i) {
+ for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup* g = info->groups + i;
gRes[gResIdx] = calloc(1, sizeof(SFilterGroupCtx));
@@ -2141,9 +2266,9 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
colIdxi = 0;
empty = false;
- for (uint16_t j = 0; j < g->unitNum; ++j) {
+ for (uint32_t j = 0; j < g->unitNum; ++j) {
SFilterUnit* u = FILTER_GROUP_UNIT(info, g, j);
- uint16_t cidx = FILTER_UNIT_COL_IDX(u);
+ uint32_t cidx = FILTER_UNIT_COL_IDX(u);
if (gRes[gResIdx]->colInfo[cidx].info == NULL) {
gRes[gResIdx]->colInfo[cidx].info = (SArray *)taosArrayInit(4, POINTER_BYTES);
@@ -2159,10 +2284,10 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
}
if (colIdxi > 1) {
- qsort(colIdx, colIdxi, sizeof(uint16_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
+ qsort(colIdx, colIdxi, sizeof(uint32_t), getComparFunc(TSDB_DATA_TYPE_USMALLINT, 0));
}
- for (uint16_t l = 0; l < colIdxi; ++l) {
+ for (uint32_t l = 0; l < colIdxi; ++l) {
int32_t type = gRes[gResIdx]->colInfo[colIdx[l]].dataType;
if (FILTER_NO_MERGE_DATA_TYPE(type)) {
@@ -2201,7 +2326,7 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t
}
void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) {
- uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0;
+ uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
bool equal = false;
for (; m < gRes1->colNum; ++m) {
@@ -2224,6 +2349,15 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
*conflict = true;
return;
}
+
+ // for long in operation
+ if (gRes1->colInfo[idx1].optr == TSDB_RELATION_EQUAL && gRes2->colInfo[idx2].optr == TSDB_RELATION_EQUAL) {
+ SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info;
+ if (ctx->pCompareFunc(&gRes1->colInfo[idx1].value, &gRes2->colInfo[idx2].value)){
+ *conflict = true;
+ return;
+ }
+ }
++n;
equal = true;
@@ -2241,7 +2375,7 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool
}
-int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint16_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
+int32_t filterMergeTwoGroupsImpl(SFilterInfo *info, SFilterRangeCtx **ctx, int32_t optr, uint32_t cidx, SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *empty, bool *all) {
SFilterField *fi = FILTER_GET_COL_FIELD(info, cidx);
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
@@ -2271,10 +2405,10 @@ int32_t filterMergeTwoGroups(SFilterInfo *info, SFilterGroupCtx** gRes1, SFilter
FILTER_SET_FLAG(info->status, FI_STATUS_REWRITE);
- uint16_t idx1 = 0, idx2 = 0, m = 0, n = 0;
+ uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0;
bool numEqual = (*gRes1)->colNum == (*gRes2)->colNum;
bool equal = false;
- uint16_t equal1 = 0, equal2 = 0, merNum = 0;
+ uint32_t equal1 = 0, equal2 = 0, merNum = 0;
SFilterRangeCtx *ctx = NULL;
SFilterColCtx colCtx = {0};
SArray* colCtxs = taosArrayInit((*gRes2)->colNum, sizeof(SFilterColCtx));
@@ -2388,7 +2522,7 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
qsort(gRes, *gResNum, POINTER_BYTES, filterCompareGroupCtx);
int32_t pEnd = 0, cStart = 0, cEnd = 0;
- uint16_t pColNum = 0, cColNum = 0;
+ uint32_t pColNum = 0, cColNum = 0;
int32_t movedNum = 0;
bool all = false;
@@ -2454,7 +2588,6 @@ int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t *gR
}
cStart = i;
- cEnd = i;
cColNum = gRes[i]->colNum;
}
@@ -2470,7 +2603,7 @@ _return:
int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
size_t groupSize = taosArrayGetSize(group);
- info->groupNum = (uint16_t)groupSize;
+ info->groupNum = (uint32_t)groupSize;
if (info->groupNum > 0) {
info->groups = calloc(info->groupNum, sizeof(*info->groups));
@@ -2487,7 +2620,7 @@ int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray* group) {
int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
if (!FILTER_GET_FLAG(info->status, FI_STATUS_REWRITE)) {
- //qDebug("no need rewrite");
+ qDebug("no need rewrite");
return TSDB_CODE_SUCCESS;
}
@@ -2499,7 +2632,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
SFilterGroupCtx *res = NULL;
SFilterColInfo *colInfo = NULL;
int32_t optr = 0;
- uint16_t uidx = 0;
+ uint32_t uidx = 0;
memset(info, 0, sizeof(*info));
@@ -2519,7 +2652,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
SFilterGroup ng = {0};
- for (uint16_t m = 0; m < res->colNum; ++m) {
+ for (uint32_t m = 0; m < res->colNum; ++m) {
colInfo = &res->colInfo[res->colIdx[m]];
if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
assert(colInfo->type == RANGE_TYPE_UNIT);
@@ -2555,13 +2688,13 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum
}
int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t gResNum) {
- uint16_t *idxs = NULL;
- uint16_t colNum = 0;
+ uint32_t *idxs = NULL;
+ uint32_t colNum = 0;
SFilterGroupCtx *res = NULL;
- uint16_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
+ uint32_t *idxNum = calloc(info->fields[FLD_TYPE_COLUMN].num, sizeof(*idxNum));
for (int32_t i = 0; i < gResNum; ++i) {
- for (uint16_t m = 0; m < gRes[i]->colNum; ++m) {
+ for (uint32_t m = 0; m < gRes[i]->colNum; ++m) {
SFilterColInfo *colInfo = &gRes[i]->colInfo[gRes[i]->colIdx[m]];
if (FILTER_NO_MERGE_DATA_TYPE(colInfo->dataType)) {
continue;
@@ -2571,7 +2704,7 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
}
}
- for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+ for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
if (idxNum[i] < gResNum) {
continue;
}
@@ -2592,9 +2725,9 @@ int32_t filterGenerateColRange(SFilterInfo *info, SFilterGroupCtx** gRes, int32_
for (int32_t i = 0; i < gResNum; ++i) {
res = gRes[i];
- uint16_t n = 0;
+ uint32_t n = 0;
- for (uint16_t m = 0; m < info->colRangeNum; ++m) {
+ for (uint32_t m = 0; m < info->colRangeNum; ++m) {
for (; n < res->colNum; ++n) {
if (res->colIdx[n] < idxs[m]) {
continue;
@@ -2642,7 +2775,7 @@ _return:
}
int32_t filterPostProcessRange(SFilterInfo *info) {
- for (uint16_t i = 0; i < info->colRangeNum; ++i) {
+ for (uint32_t i = 0; i < info->colRangeNum; ++i) {
SFilterRangeCtx* ctx = info->colRange[i];
SFilterRangeNode *r = ctx->rs;
while (r) {
@@ -2656,13 +2789,11 @@ int32_t filterPostProcessRange(SFilterInfo *info) {
int32_t filterGenerateComInfo(SFilterInfo *info) {
- uint16_t n = 0;
-
info->cunits = malloc(info->unitNum * sizeof(*info->cunits));
info->blkUnitRes = malloc(sizeof(*info->blkUnitRes) * info->unitNum);
info->blkUnits = malloc(sizeof(*info->blkUnits) * (info->unitNum + 1) * info->groupNum);
- for (uint16_t i = 0; i < info->unitNum; ++i) {
+ for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
info->cunits[i].func = filterGetCompFuncIdx(FILTER_UNIT_DATA_TYPE(unit), unit->compare.optr);
@@ -2672,7 +2803,11 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].colId = FILTER_UNIT_COL_ID(info, unit);
if (unit->right.type == FLD_TYPE_VALUE) {
- info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
+ if(FILTER_UNIT_DATA_TYPE(unit) == TSDB_DATA_TYPE_JSON){ // json value is tVariant
+ info->cunits[i].valData = FILTER_UNIT_JSON_VAL_DATA(info, unit);
+ }else{
+ info->cunits[i].valData = FILTER_UNIT_VAL_DATA(info, unit);
+ }
} else {
info->cunits[i].valData = NULL;
}
@@ -2685,30 +2820,12 @@ int32_t filterGenerateComInfo(SFilterInfo *info) {
info->cunits[i].dataSize = FILTER_UNIT_COL_SIZE(info, unit);
info->cunits[i].dataType = FILTER_UNIT_DATA_TYPE(unit);
}
-
- uint16_t cgroupNum = info->groupNum + 1;
-
- for (uint16_t i = 0; i < info->groupNum; ++i) {
- cgroupNum += info->groups[i].unitNum;
- }
-
- info->cgroups = malloc(cgroupNum * sizeof(*info->cgroups));
-
- for (uint16_t i = 0; i < info->groupNum; ++i) {
- info->cgroups[n++] = info->groups[i].unitNum;
-
- for (uint16_t m = 0; m < info->groups[i].unitNum; ++m) {
- info->cgroups[n++] = info->groups[i].unitIdxs[m];
- }
- }
-
- info->cgroups[n] = 0;
return TSDB_CODE_SUCCESS;
}
int32_t filterUpdateComUnits(SFilterInfo *info) {
- for (uint16_t i = 0; i < info->unitNum; ++i) {
+ for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
@@ -2723,7 +2840,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
memset(info->blkUnitRes, 0, sizeof(*info->blkUnitRes) * info->unitNum);
- for (int32_t k = 0; k < info->unitNum; ++k) {
+ for (uint32_t k = 0; k < info->unitNum; ++k) {
int32_t index = -1;
SFilterComUnit *cunit = &info->cunits[k];
@@ -2768,18 +2885,20 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
}
}
- if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
+ if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL
|| cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
- || cunit->optr == TSDB_RELATION_NOT_EQUAL) {
+ || cunit->optr == TSDB_RELATION_NOT_EQUAL || cunit->optr == TSDB_RELATION_CONTAINS) {
continue;
}
SColumnDataAgg* pDataBlockst = &pDataStatis[index];
void *minVal, *maxVal;
+ float minv = 0;
+ float maxv = 0;
if (cunit->dataType == TSDB_DATA_TYPE_FLOAT) {
- float minv = (float)(*(double *)(&pDataBlockst->min));
- float maxv = (float)(*(double *)(&pDataBlockst->max));
+ minv = (float)(*(double *)(&pDataBlockst->min));
+ maxv = (float)(*(double *)(&pDataBlockst->max));
minVal = &minv;
maxVal = &maxv;
@@ -2834,12 +2953,15 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
}
-// CHK_LRET(rmUnit == 0, TSDB_CODE_SUCCESS, "NO Block Filter APPLY");
+ if (rmUnit == 0) {
+ fltDebug("NO Block Filter APPLY");
+ FLT_RET(TSDB_CODE_SUCCESS);
+ }
info->blkGroupNum = info->groupNum;
- uint16_t *unitNum = info->blkUnits;
- uint16_t *unitIdx = unitNum + 1;
+ uint32_t *unitNum = info->blkUnits;
+ uint32_t *unitIdx = unitNum + 1;
int32_t all = 0, empty = 0;
for (uint32_t g = 0; g < info->groupNum; ++g) {
@@ -2849,7 +2971,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3
empty = 0;
for (uint32_t u = 0; u < group->unitNum; ++u) {
- uint16_t uidx = group->unitIdxs[u];
+ uint32_t uidx = group->unitIdxs[u];
if (info->blkUnitRes[uidx] == 1) {
--(*unitNum);
all = 1;
@@ -2896,17 +3018,19 @@ _return:
bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true;
- uint16_t *unitIdx = NULL;
-
- *p = calloc(numOfRows, sizeof(int8_t));
+ uint32_t *unitIdx = NULL;
+ if (*p == NULL) {
+ *p = calloc(numOfRows, sizeof(int8_t));
+ }
+
for (int32_t i = 0; i < numOfRows; ++i) {
//FILTER_UNIT_CLR_F(info);
unitIdx = info->blkUnits;
for (uint32_t g = 0; g < info->blkGroupNum; ++g) {
- uint16_t unitNum = *(unitIdx++);
+ uint32_t unitNum = *(unitIdx++);
for (uint32_t u = 0; u < unitNum; ++u) {
SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)];
void *colData = (char *)cunit->colData + cunit->dataSize * i;
@@ -3001,12 +3125,25 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
return all;
}
- *p = calloc(numOfRows, sizeof(int8_t));
+ if (*p == NULL) {
+ *p = calloc(numOfRows, sizeof(int8_t));
+ }
for (int32_t i = 0; i < numOfRows; ++i) {
- uint16_t uidx = info->groups[0].unitIdxs[0];
+ uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
- (*p)[i] = isNull(colData, info->cunits[uidx].dataType);
+ if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
+ if (!colData){ // for json->'key' is null
+ (*p)[i] = 1;
+ }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null
+ colData = POINTER_SHIFT(colData, CHAR_BYTES);
+ (*p)[i] = isNull(colData, info->cunits[uidx].dataType);
+ }else{
+ (*p)[i] = 0;
+ }
+ }else{
+ (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType));
+ }
if ((*p)[i] == 0) {
all = false;
}
@@ -3022,12 +3159,27 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
return all;
}
- *p = calloc(numOfRows, sizeof(int8_t));
-
+ if (*p == NULL) {
+ *p = calloc(numOfRows, sizeof(int8_t));
+ }
+
for (int32_t i = 0; i < numOfRows; ++i) {
- uint16_t uidx = info->groups[0].unitIdxs[0];
+ uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
- (*p)[i] = !isNull(colData, info->cunits[uidx].dataType);
+
+ if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
+ if (!colData) { // for json->'key' is not null
+ (*p)[i] = 0;
+ }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null
+ colData = POINTER_SHIFT(colData, CHAR_BYTES);
+ (*p)[i] = !isNull(colData, info->cunits[uidx].dataType);
+ }else{ // for json->'key' is not null
+ (*p)[i] = 1;
+ }
+ }else {
+ (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType));
+ }
+
if ((*p)[i] == 0) {
all = false;
}
@@ -3036,7 +3188,48 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
return all;
}
-bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
+static void doJsonCompare(SFilterComUnit *cunit, int8_t *result, void* colData){
+ if(cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH){
+ uint8_t jsonType = *(char*)colData;
+ char* realData = POINTER_SHIFT(colData, CHAR_BYTES);
+ if (jsonType != TSDB_DATA_TYPE_NCHAR){
+ *result = false;
+ }else{
+ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+ int len = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), varDataVal(newColData));
+ if (len < 0){
+ qError("castConvert1 taosUcs4ToMbs error");
+ tfree(newColData);
+ return;
+ }
+ varDataSetLen(newColData, len);
+ tVariant* val = cunit->valData;
+ char newValData[TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE] = {0};
+ assert(val->nLen <= TSDB_REGEX_STRING_DEFAULT_LEN * TSDB_NCHAR_SIZE);
+ memcpy(varDataVal(newValData), val->pz, val->nLen);
+ varDataSetLen(newValData, val->nLen);
+ *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, newValData);
+ tfree(newColData);
+ }
+ }else if(cunit->optr == TSDB_RELATION_LIKE){
+ uint8_t jsonType = *(char*)colData;
+ char* realData = POINTER_SHIFT(colData, CHAR_BYTES);
+ if (jsonType != TSDB_DATA_TYPE_NCHAR){
+ *result = false;
+ }else{
+ tVariant* val = cunit->valData;
+ char* newValData = calloc(val->nLen + VARSTR_HEADER_SIZE, 1);
+ memcpy(varDataVal(newValData), val->pz, val->nLen);
+ varDataSetLen(newValData, val->nLen);
+ *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, realData, newValData);
+ tfree(newValData);
+ }
+ }else{
+ *result = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
+ }
+}
+
+bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols) {
SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true;
uint16_t dataSize = info->cunits[0].dataSize;
@@ -3050,10 +3243,12 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
return all;
}
- *p = calloc(numOfRows, sizeof(int8_t));
+ if (*p == NULL) {
+ *p = calloc(numOfRows, sizeof(int8_t));
+ }
for (int32_t i = 0; i < numOfRows; ++i) {
- if (isNull(colData, info->cunits[0].dataType)) {
+ if (colData == NULL || isNull(colData, info->cunits[0].dataType)) {
all = false;
colData += dataSize;
continue;
@@ -3078,19 +3273,37 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
if (filterExecuteBasedOnStatis(info, numOfRows, p, statis, numOfCols, &all) == 0) {
return all;
}
-
- *p = calloc(numOfRows, sizeof(int8_t));
+
+ if (*p == NULL) {
+ *p = calloc(numOfRows, sizeof(int8_t));
+ }
for (int32_t i = 0; i < numOfRows; ++i) {
- uint16_t uidx = info->groups[0].unitIdxs[0];
+ uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
- if (isNull(colData, info->cunits[uidx].dataType)) {
+ if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
+ (*p)[i] = 0;
all = false;
continue;
}
+ // match/nmatch for nchar type need convert from ucs4 to mbs
+
+ if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_NCHAR && (info->cunits[uidx].optr == TSDB_RELATION_MATCH || info->cunits[uidx].optr == TSDB_RELATION_NMATCH)){
+ char *newColData = calloc(info->cunits[uidx].dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+ int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+ if (len < 0){
+ qError("castConvert1 taosUcs4ToMbs error");
+ }else{
+ varDataSetLen(newColData, len);
+ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, newColData, info->cunits[uidx].valData);
+ }
+ tfree(newColData);
+ }else if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
+ doJsonCompare(&(info->cunits[uidx]), &(*p)[i], colData);
+ }else{
+ (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
+ }
- (*p)[i] = filterDoCompare(gDataCompare[info->cunits[uidx].func], info->cunits[uidx].optr, colData, info->cunits[uidx].valData);
-
if ((*p)[i] == 0) {
all = false;
}
@@ -3108,15 +3321,17 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
return all;
}
- *p = calloc(numOfRows, sizeof(int8_t));
-
+ if (*p == NULL) {
+ *p = calloc(numOfRows, sizeof(int8_t));
+ }
+
for (int32_t i = 0; i < numOfRows; ++i) {
//FILTER_UNIT_CLR_F(info);
for (uint32_t g = 0; g < info->groupNum; ++g) {
SFilterGroup *group = &info->groups[g];
for (uint32_t u = 0; u < group->unitNum; ++u) {
- uint16_t uidx = group->unitIdxs[u];
+ uint32_t uidx = group->unitIdxs[u];
SFilterComUnit *cunit = &info->cunits[uidx];
void *colData = (char *)cunit->colData + cunit->dataSize * i;
@@ -3125,7 +3340,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
//} else {
uint8_t optr = cunit->optr;
- if (isNull(colData, cunit->dataType)) {
+ if (colData == NULL || isNull(colData, cunit->dataType)) {
(*p)[i] = optr == TSDB_RELATION_ISNULL ? true : false;
} else {
if (optr == TSDB_RELATION_NOTNULL) {
@@ -3135,7 +3350,21 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
} else if (cunit->rfunc >= 0) {
(*p)[i] = (*gRangeCompare[cunit->rfunc])(colData, colData, cunit->valData, cunit->valData2, gDataCompare[cunit->func]);
} else {
- (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
+ if(cunit->dataType == TSDB_DATA_TYPE_NCHAR && (cunit->optr == TSDB_RELATION_MATCH || cunit->optr == TSDB_RELATION_NMATCH)){
+ char *newColData = calloc(cunit->dataSize * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 1);
+ int32_t len = taosUcs4ToMbs(varDataVal(colData), varDataLen(colData), varDataVal(newColData));
+ if (len < 0){
+ qError("castConvert1 taosUcs4ToMbs error");
+ }else{
+ varDataSetLen(newColData, len);
+ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, newColData, cunit->valData);
+ }
+ tfree(newColData);
+ }else if(cunit->dataType == TSDB_DATA_TYPE_JSON){
+ doJsonCompare(cunit, &(*p)[i], colData);
+ }else{
+ (*p)[i] = filterDoCompare(gDataCompare[cunit->func], cunit->optr, colData, cunit->valData);
+ }
}
//FILTER_UNIT_SET_R(info, uidx, p[i]);
@@ -3211,13 +3440,13 @@ int32_t filterPreprocess(SFilterInfo *info) {
filterMergeGroups(info, gRes, &gResNum);
if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) {
-// qInfo("Final - FilterInfo: [ALL]");
+ qInfo("Final - FilterInfo: [ALL]");
goto _return;
}
if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
-// qInfo("Final - FilterInfo: [EMPTY]");
+ qInfo("Final - FilterInfo: [EMPTY]");
goto _return;
}
@@ -3244,9 +3473,29 @@ _return:
return TSDB_CODE_SUCCESS;
}
-int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock) {
-// CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
-// CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
+int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
+ CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
+ CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
+
+ if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+ SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
+ SSchema* sch = fi->desc;
+
+ (*fp)(param, sch->colId, &fi->data);
+ }
+
+ filterUpdateComUnits(info);
+
+ return TSDB_CODE_SUCCESS;
+}
+
+int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) {
+ CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL");
+ CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds");
if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) {
return TSDB_CODE_SUCCESS;
@@ -3255,15 +3504,8 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat
for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema* sch = fi->desc;
-
- for (int32_t j = 0; j < numOfCols; ++j) {
- SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);
- if (sch->colId == pColInfo->info.colId) {
- fi->data = pColInfo->pData;
-
- break;
- }
- }
+
+ (*fp)(param, sch->colId, sch->name, &fi->data);
}
filterUpdateComUnits(info);
@@ -3271,46 +3513,32 @@ int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDat
return TSDB_CODE_SUCCESS;
}
-
-int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options) {
+int32_t fltInitFromNode(SNode* tree, SFilterInfo *info, uint32_t options) {
int32_t code = TSDB_CODE_SUCCESS;
- SFilterInfo *info = NULL;
-
-// CHK_LRET(tree == NULL || pinfo == NULL, TSDB_CODE_QRY_APP_ERROR, "invalid param");
-
- if (*pinfo == NULL) {
- *pinfo = calloc(1, sizeof(SFilterInfo));
- }
-
- info = *pinfo;
-
- info->options = options;
SArray* group = taosArrayInit(FILTER_DEFAULT_GROUP_SIZE, sizeof(SFilterGroup));
filterInitUnitsFields(info);
- code = filterTreeToGroup(tree, info, group);
-
- ERR_JRET(code);
+ SFltBuildGroupCtx tctx = {.info = info, .group = group};
+ nodesWalkNode(tree, fltTreeToGroup, (void *)&tctx);
+ FLT_ERR_JRET(tctx.code);
filterConvertGroupFromArray(info, group);
+ taosArrayDestroy(group);
- ERR_JRET(filterInitValFieldData(info));
+ FLT_ERR_JRET(filterInitValFieldData(info));
if (!FILTER_GET_FLAG(info->options, FI_OPTION_NO_REWRITE)) {
filterDumpInfoToString(info, "Before preprocess", 0);
- ERR_JRET(filterPreprocess(info));
+ FLT_ERR_JRET(filterPreprocess(info));
- CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
+ FLT_CHK_JMP(FILTER_GET_FLAG(info->status, FI_STATUS_ALL));
if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) {
- taosArrayDestroy(group);
return code;
}
-
- //ERR_JRET(filterInitUnitFunc(info));
}
info->unitRes = malloc(info->unitNum * sizeof(*info->unitRes));
@@ -3318,18 +3546,12 @@ int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t option
filterDumpInfoToString(info, "Final", 0);
- taosArrayDestroy(group);
-
return code;
_return:
-// qInfo("No filter, code:%d", code);
+
+ qInfo("No filter, code:%d", code);
- taosArrayDestroy(group);
- filterFreeInfo(*pinfo);
-
- *pinfo = NULL;
-
return code;
}
@@ -3348,7 +3570,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
bool ret = true;
void *minVal, *maxVal;
- for (int32_t k = 0; k < info->colRangeNum; ++k) {
+ for (uint32_t k = 0; k < info->colRangeNum; ++k) {
int32_t index = -1;
SFilterRangeCtx *ctx = info->colRange[k];
for(int32_t i = 0; i < numOfCols; ++i) {
@@ -3368,30 +3590,35 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
break;
}
- if ((pDataStatis[index].numOfNull <= 0) && (ctx->isnull && !ctx->notnull && !ctx->isrange)) {
- ret = false;
- break;
- }
-
- // all data in current column are NULL, no need to check its boundary value
- if (pDataStatis[index].numOfNull == numOfRows) {
-
- // if isNULL query exists, load the null data column
- if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) {
+ if (pDataStatis[index].numOfNull <= 0) {
+ if (ctx->isnull && !ctx->notnull && !ctx->isrange) {
ret = false;
break;
}
+ } else if (pDataStatis[index].numOfNull > 0) {
+ if (pDataStatis[index].numOfNull == numOfRows) {
+ if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) {
+ ret = false;
+ break;
+ }
- continue;
+ continue;
+ } else {
+ if (ctx->isnull) {
+ continue;
+ }
+ }
}
SColumnDataAgg* pDataBlockst = &pDataStatis[index];
SFilterRangeNode *r = ctx->rs;
+ float minv = 0;
+ float maxv = 0;
if (ctx->type == TSDB_DATA_TYPE_FLOAT) {
- float minv = (float)(*(double *)(&pDataBlockst->min));
- float maxv = (float)(*(double *)(&pDataBlockst->max));
+ minv = (float)(*(double *)(&pDataBlockst->min));
+ maxv = (float)(*(double *)(&pDataBlockst->max));
minVal = &minv;
maxVal = &maxv;
@@ -3408,7 +3635,9 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t
r = r->next;
}
- CHK_RET(!ret, ret);
+ if (!ret) {
+ return ret;
+ }
}
return ret;
@@ -3426,7 +3655,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
int32_t code = 0;
bool empty = false, all = false;
- for (int32_t i = 0; i < info->groupNum; ++i) {
+ for (uint32_t i = 0; i < info->groupNum; ++i) {
SFilterGroup *group = &info->groups[i];
if (group->unitNum > 1) {
cur = tmpc;
@@ -3436,8 +3665,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
optr = TSDB_RELATION_OR;
}
- for (int32_t u = 0; u < group->unitNum; ++u) {
- uint16_t uidx = group->unitIdxs[u];
+ for (uint32_t u = 0; u < group->unitNum; ++u) {
+ uint32_t uidx = group->unitIdxs[u];
SFilterUnit *unit = &info->units[uidx];
uint8_t raOptr = FILTER_UNIT_OPTR(unit);
@@ -3478,8 +3707,8 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
} else {
filterGetRangeNum(prev, &num);
if (num > 1) {
- //qError("only one time range accepted, num:%d", num);
- ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
+ qError("only one time range accepted, num:%d", num);
+ FLT_ERR_JRET(TSDB_CODE_QRY_INVALID_TIME_CONDITION);
}
CHK_JMP(num < 1);
@@ -3493,7 +3722,7 @@ int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win) {
filterFreeRangeCtx(prev);
filterFreeRangeCtx(tmpc);
- //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
+ qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
return TSDB_CODE_SUCCESS;
_return:
@@ -3503,14 +3732,18 @@ _return:
filterFreeRangeCtx(prev);
filterFreeRangeCtx(tmpc);
- //qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
+ qDebug("qFilter time range:[%"PRId64 "]-[%"PRId64 "]", win->skey, win->ekey);
return code;
}
int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar) {
- for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+ if (FILTER_EMPTY_RES(info) || FILTER_ALL_RES(info)) {
+ return TSDB_CODE_SUCCESS;
+ }
+
+ for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
if (type == TSDB_DATA_TYPE_NCHAR) {
@@ -3523,7 +3756,20 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
char *src = FILTER_GET_COL_FIELD_DATA(fi, j);
char *dst = FILTER_GET_COL_FIELD_DATA(&nfi, j);
int32_t len = 0;
- taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
+ char *varSrc = varDataVal(src);
+ size_t k = 0, varSrcLen = varDataLen(src);
+ while (k < varSrcLen && varSrc[k++] == -1) {}
+ if (k == varSrcLen) {
+ /* NULL */
+ varDataLen(dst) = (VarDataLenT) varSrcLen;
+ varDataCopy(dst, src);
+ continue;
+ }
+ bool ret = taosMbsToUcs4(varDataVal(src), varDataLen(src), varDataVal(dst), bufSize, &len);
+ if(!ret) {
+ qError("filterConverNcharColumns taosMbsToUcs4 error");
+ return TSDB_CODE_FAILED;
+ }
varDataLen(dst) = len;
}
@@ -3541,7 +3787,7 @@ int32_t filterConverNcharColumns(SFilterInfo* info, int32_t rows, bool *gotNchar
}
int32_t filterFreeNcharColumns(SFilterInfo* info) {
- for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
+ for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
int32_t type = FILTER_GET_COL_FIELD_TYPE(fi);
if (type == TSDB_DATA_TYPE_NCHAR) {
@@ -3552,7 +3798,74 @@ int32_t filterFreeNcharColumns(SFilterInfo* info) {
return TSDB_CODE_SUCCESS;
}
-
+##################################################################################
+
+EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
+ SFltTreeStat *stat = (SFltTreeStat *)pContext;
+}
+
+int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
+ nodesRewriteNodePostOrder(pNode, fltReviseRewriter, (void *)pStat);
+
+ FLT_RET(pStat->code);
+}
+
+int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
+
+}
+
+
+int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) {
+ int32_t code = 0;
+ SFilterInfo *info = NULL;
+
+ if (pNode == NULL || pInfo == NULL) {
+ fltError("invalid param");
+ FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
+ }
+
+ if (*pInfo == NULL) {
+ *pInfo = calloc(1, sizeof(SFilterInfo));
+ if (NULL == *pInfo) {
+ fltError("calloc %d failed", (int32_t)sizeof(SFilterInfo));
+ FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+ }
+ }
+
+ info = *pInfo;
+ info->options = options;
+
+ SFltTreeStat stat = {0};
+ FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat));
+
+ if (!info->scalarMode) {
+ FLT_ERR_JRET(fltInitFromNode(pNode, info, options));
+ }
+
+ FLT_ERR_JRET(fltOptimizeNodes(info, &pNode, &stat));
+
+ return code;
+
+_return:
+
+ filterFreeInfo(*pInfo);
+
+ *pInfo = NULL;
+
+ FLT_RET(code);
+}
+
+FORCE_INLINE bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) {
+ if (info->scalarMode) {
+ SScalarParam output = {0};
+ FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output));
+
+ *p = output.data;
+ return TSDB_CODE_SUCCESS;
+ }
+
+ return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols);
+}
diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index 082dd6cfd4..d13589545a 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -45,6 +45,11 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
break;
}
+ case QUERY_NODE_NODE_LIST: {
+ SNodeListNode *nodeList = (SNodeListNode *)node;
+ //TODO BUILD HASH
+ break;
+ }
case QUERY_NODE_COLUMN_REF: {
if (NULL == ctx) {
sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 07f32c87d8..57e9bc0d09 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -19,7 +19,7 @@
#include "sclvector.h"
#include "tcompare.h"
#include "querynodes.h"
-#include "filter.h"
+#include "filterInt.h"
#include "query.h"
//GET_TYPED_DATA(v, double, pRight->type, (char *)&((right)[i]));