From 2f87a6ee443800fe015cedad2fd87e9ecef6a445 Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 15 Jul 2021 13:23:34 +0800 Subject: [PATCH] support in --- src/query/src/qFilter.c | 185 +++++++++++++++------- src/util/inc/hash.h | 2 + src/util/src/hash.c | 6 + tests/script/general/parser/condition.sim | 17 ++ 4 files changed, 154 insertions(+), 56 deletions(-) diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 95bbdc5677..627f7eb13f 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -585,13 +585,55 @@ int32_t filterAddUnitToGroup(SFilterGroup *group, uint16_t unitIdx) { -int32_t filterAddUnitFromNode(SFilterInfo *info, tExprNode* tree) { +int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *group) { SFilterFieldId left = {0}, right = {0}; - + filterAddFieldFromNode(info, tree->_node.pLeft, &left); - filterAddFieldFromNode(info, tree->_node.pRight, &right); - - filterAddUnit(info, tree->_node.optr, &left, &right); + + tVariant* var = tree->_node.pRight->pVal; + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(info, left)); + + if (tree->_node.optr == TSDB_RELATION_IN && (!IS_VAR_DATA_TYPE(type))) { + void *data = NULL; + convertFilterSetFromBinary((void **)&data, var->pz, var->nLen, type); + CHK_LRET(data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); + + void *p = taosHashIterate((SHashObj *)data, NULL); + while(p) { + void *key = taosHashGetDataKey((SHashObj *)data, p); + void *fdata = NULL; + + if (IS_VAR_DATA_TYPE(type)) { + uint32_t len = taosHashGetDataKeyLen((SHashObj *)data, p); + fdata = malloc(len + VARSTR_HEADER_SIZE); + varDataLen(fdata) = len; + memcpy(varDataVal(fdata), key, len); + } else { + fdata = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(fdata, key); + } + + filterAddField(info, NULL, fdata, FLD_TYPE_VALUE, &right); + + filterAddUnit(info, TSDB_RELATION_EQUAL, &left, &right); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, info->unitNum - 1); + + taosArrayPush(group, &fgroup); + + p = taosHashIterate((SHashObj *)data, p); + } + } else { + filterAddFieldFromNode(info, tree->_node.pRight, &right); + + filterAddUnit(info, tree->_node.optr, &left, &right); + + SFilterGroup fgroup = {0}; + filterAddUnitToGroup(&fgroup, info->unitNum - 1); + + taosArrayPush(group, &fgroup); + } return TSDB_CODE_SUCCESS; } @@ -611,6 +653,7 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit* u return filterAddUnit(dst, FILTER_UNIT_OPTR(u), &left, &right); } + int32_t filterAddGroupUnitFromRange(SFilterInfo *dst, SFilterInfo *src, SFilterColRange *cra, SFilterGroup *g, int32_t optr, SArray *res) { SFilterFieldId left, right; @@ -633,53 +676,88 @@ int32_t filterAddGroupUnitFromRange(SFilterInfo *dst, SFilterInfo *src, SFilterC assert(!((FILTER_GET_FLAG(cra->ra.sflag, RA_NULL)) && (FILTER_GET_FLAG(cra->ra.eflag, RA_NULL)))); + if ((!FILTER_GET_FLAG(cra->ra.sflag, RA_NULL)) &&(!FILTER_GET_FLAG(cra->ra.eflag, RA_NULL))) { + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); + __compar_fn_t func = getComparFunc(type, 0); + if (func(&cra->ra.s, &cra->ra.e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &cra->ra.s); + filterAddField(dst, NULL, data, FLD_TYPE_VALUE, &right); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right); + return filterAddUnitToGroup(g, dst->unitNum - 1); + } + } + if (!FILTER_GET_FLAG(cra->ra.sflag, RA_NULL)) { - filterAddField(dst, NULL, &cra->ra.s, FLD_TYPE_VALUE, &right); + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &cra->ra.s); + filterAddField(dst, NULL, data, FLD_TYPE_VALUE, &right); filterAddUnit(dst, FILTER_GET_FLAG(cra->ra.sflag, RA_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right); filterAddUnitToGroup(g, dst->unitNum - 1); } if (!FILTER_GET_FLAG(cra->ra.eflag, RA_NULL)) { - filterAddField(dst, NULL, &cra->ra.e, FLD_TYPE_VALUE, &right); + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &cra->ra.e); + filterAddField(dst, NULL, data, FLD_TYPE_VALUE, &right); filterAddUnit(dst, FILTER_GET_FLAG(cra->ra.eflag, RA_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right); filterAddUnitToGroup(g, dst->unitNum - 1); } - } else { - SFilterGroup ng = {0}; - g = &ng; - - if (cra->isNull) { - filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL); - filterAddUnitToGroup(g, dst->unitNum - 1); - taosArrayPush(res, g); - } - - if (cra->notNull) { - memset(g, 0, sizeof(*g)); - - filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL); - filterAddUnitToGroup(g, dst->unitNum - 1); - taosArrayPush(res, g); - } + return TSDB_CODE_SUCCESS; + } + + // OR PROCESS + + SFilterGroup ng = {0}; + g = &ng; + + if (cra->isNull) { + filterAddUnit(dst, TSDB_RELATION_ISNULL, &left, NULL); + filterAddUnitToGroup(g, dst->unitNum - 1); + taosArrayPush(res, g); + } + + if (cra->notNull) { memset(g, 0, sizeof(*g)); - if (!FILTER_GET_FLAG(cra->ra.sflag, RA_NULL)) { - filterAddField(dst, NULL, &cra->ra.s, FLD_TYPE_VALUE, &right); - filterAddUnit(dst, FILTER_GET_FLAG(cra->ra.sflag, RA_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right); - filterAddUnitToGroup(g, dst->unitNum - 1); - } - - if (!FILTER_GET_FLAG(cra->ra.eflag, RA_NULL)) { - filterAddField(dst, NULL, &cra->ra.e, FLD_TYPE_VALUE, &right); - filterAddUnit(dst, FILTER_GET_FLAG(cra->ra.eflag, RA_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right); - filterAddUnitToGroup(g, dst->unitNum - 1); - } + filterAddUnit(dst, TSDB_RELATION_NOTNULL, &left, NULL); + filterAddUnitToGroup(g, dst->unitNum - 1); + taosArrayPush(res, g); + } - if (g->unitNum > 0) { + memset(g, 0, sizeof(*g)); + + if ((!FILTER_GET_FLAG(cra->ra.sflag, RA_NULL)) &&(!FILTER_GET_FLAG(cra->ra.eflag, RA_NULL))) { + int32_t type = FILTER_GET_COL_FIELD_TYPE(FILTER_GET_FIELD(dst, left)); + __compar_fn_t func = getComparFunc(type, 0); + if (func(&cra->ra.s, &cra->ra.e) == 0) { + void *data = malloc(sizeof(int64_t)); + SIMPLE_COPY_VALUES(data, &cra->ra.s); + filterAddField(dst, NULL, data, FLD_TYPE_VALUE, &right); + filterAddUnit(dst, TSDB_RELATION_EQUAL, &left, &right); + filterAddUnitToGroup(g, dst->unitNum - 1); + taosArrayPush(res, g); + return TSDB_CODE_SUCCESS; } } + + if (!FILTER_GET_FLAG(cra->ra.sflag, RA_NULL)) { + filterAddField(dst, NULL, &cra->ra.s, FLD_TYPE_VALUE, &right); + filterAddUnit(dst, FILTER_GET_FLAG(cra->ra.sflag, RA_EXCLUDE) ? TSDB_RELATION_GREATER : TSDB_RELATION_GREATER_EQUAL, &left, &right); + filterAddUnitToGroup(g, dst->unitNum - 1); + } + + if (!FILTER_GET_FLAG(cra->ra.eflag, RA_NULL)) { + filterAddField(dst, NULL, &cra->ra.e, FLD_TYPE_VALUE, &right); + filterAddUnit(dst, FILTER_GET_FLAG(cra->ra.eflag, RA_EXCLUDE) ? TSDB_RELATION_LESS : TSDB_RELATION_LESS_EQUAL, &left, &right); + filterAddUnitToGroup(g, dst->unitNum - 1); + } + + if (g->unitNum > 0) { + taosArrayPush(res, g); + } return TSDB_CODE_SUCCESS; } @@ -725,12 +803,8 @@ int32_t filterTreeToGroup(tExprNode* tree, SFilterInfo *info, SArray* group) { return TSDB_CODE_SUCCESS; } - filterAddUnitFromNode(info, tree); + filterAddGroupUnitFromNode(info, tree, group); - SFilterGroup fgroup = {0}; - filterAddUnitToGroup(&fgroup, info->unitNum - 1); - - taosArrayPush(group, &fgroup); _err_return: @@ -778,15 +852,21 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg) { SFilterUnit *unit = &info->units[i]; int32_t type = FILTER_UNIT_DATA_TYPE(unit); int32_t len = 0; - char str[128]; + int32_t tlen = 0; + char str[128] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SSchema *sch = left->desc; len = sprintf(str, "UNIT[%d] => [%d][%s] %s [", i, sch->colId, sch->name, gOptrStr[unit->compare.optr].str); - if (unit->right.type== FLD_TYPE_VALUE) { + if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); - converToStr(str + len, type, right->data, 0, &len); + char *data = right->data; + if (IS_VAR_DATA_TYPE(type)) { + tlen = varDataLen(data); + data += VARSTR_HEADER_SIZE; + } + converToStr(str + len, type, data, tlen, &tlen); } else { strcat(str, "NULL"); } @@ -831,6 +911,11 @@ int32_t filterInitValFieldData(SFilterInfo *info) { tVariant* var = fi->desc; + if (var == NULL) { + assert(fi->data != NULL); + continue; + } + if (unit->compare.optr == TSDB_RELATION_IN) { convertFilterSetFromBinary((void **)&fi->data, var->pz, var->nLen, type); CHK_LRET(fi->data == NULL, TSDB_CODE_QRY_APP_ERROR, "failed to convert in param"); @@ -942,18 +1027,6 @@ int32_t filterAddUnitRange(SFilterInfo *info, SFilterUnit* u, SFilterRMCtx *ctx, SIMPLE_COPY_VALUES(&ra.s, val); SIMPLE_COPY_VALUES(&ra.e, val); break; - case TSDB_RELATION_IN: { - void *p = taosHashIterate((SHashObj *)val, NULL); - while(p) { - void *key = taosHashGetDataKey((SHashObj *)val, p); - SIMPLE_COPY_VALUES(&ra.s, key); - SIMPLE_COPY_VALUES(&ra.e, key); - filterAddMergeRange(ctx, &ra, optr); - p = taosHashIterate((SHashObj *)val, p); - } - - return TSDB_CODE_SUCCESS; - } default: assert(0); } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 79eda33e74..57c69f7beb 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -163,6 +163,8 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); void *taosHashGetDataKey(SHashObj *pHashObj, void *data); +uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index a3501d47da..27091b1fe8 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -699,6 +699,12 @@ FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { return GET_HASH_NODE_KEY(GET_HASH_PNODE(data)); } +FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { + SHashNode * node = GET_HASH_PNODE(data); + return node->keyLen; +} + + // release the pNode, return next pNode, and lock the current entry static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { diff --git a/tests/script/general/parser/condition.sim b/tests/script/general/parser/condition.sim index b6f0e76db1..54e64c01dc 100644 --- a/tests/script/general/parser/condition.sim +++ b/tests/script/general/parser/condition.sim @@ -1696,6 +1696,23 @@ if $data00 != @21-05-05 18:19:02.000@ then return -1 endi +sql select * from stb1 where (c2 in (1,2,3,4) or c2 in (11,12,13,14)) and c2 != 11 and c2 >2 and c2 != 14; +if $rows != 4 then + return -1 +endi +if $data00 != @21-05-05 18:19:02.000@ then + return -1 +endi +if $data10 != @21-05-05 18:19:03.000@ then + return -1 +endi +if $data20 != @21-05-05 18:19:05.000@ then + return -1 +endi +if $data30 != @21-05-05 18:19:06.000@ then + return -1 +endi + sql select * from stb1 where (c1 > 60 or c1 < 4 or c1 > 10 and c1 < 20 and c1 != 13 or c1 < 2 or c1 > 50) and (c1 != 51 and c1 <= 54 and c1 != 54 and c1 >=1 and c1 != 1) and (c1 >= 11 and c1 <=52 and c1 != 52 and c1 != 11); if $rows != 2 then return -1