From bdee8ab3c11b360c3d4f57d52e297acc906f0f36 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 12 May 2023 19:00:26 +0800 Subject: [PATCH 01/28] enhance: save work and sync home/office --- source/libs/planner/src/planOptimizer.c | 6 +- source/libs/scalar/src/filter.c | 156 +++++++++++++++++++++++- 2 files changed, 159 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4f8b57de5f..a6d4898bc8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2630,11 +2630,13 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { - return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions) && + LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; } if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { - return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions) && + LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; } return false; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 02a21b66ed..77534db411 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3942,8 +3942,162 @@ _return: FLT_RET(code); } +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t val; // may be int64, uint64 and double + struct { + uint32_t nData; + uint8_t *pData; + }; // maybe varchar, nchar + } datum; +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +typedef struct { + SFltSclDatum low; + SFltSclDatum high; + bool lowExcl; + bool highExcl; +} SFltSclRange; + +int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { +} + +bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0 ; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray* pts1, SArray* pts2, SArray* result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint* pt1 = taosArrayGet(pts1, i); + SFltSclPoint* pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint* pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range +int32_t fltSclMerge(SArray* pts1, SArray* pts2, bool isUnion, SArray* merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + //first merge sort pts1 and pts2 + SArray* all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint* pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + + +int32_t fltSclIntersect(SArray* pts1, SArray* pts2, SArray* merged) { + return fltSclMerge(pts1, pts2, false, merged); +} + +int32_t fltSclUnion(SArray* pts1, SArray* pts2, SArray* merged) { + return fltSclMerge(pts1, pts2, true, merged); +} + + +typedef struct { + SColumnNode* colNode; + SValueNode* valNode; + EOperatorType type; +} SFltSclScalarFunc; + +typedef struct { + SColumnNode* colNode; + +} SFltSclConstColumn; + +typedef struct { + SValueNode* value; +} SFltSclConstant; + +typedef struct { + bool useRange; + SHashObj* colRanges; + +} SFltSclOptCtx; + +static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { + SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; + +} + + int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - // TODO + SFltSclOptCtx ctx; + nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); + return TSDB_CODE_SUCCESS; } From 51fa363fd50e148ec730f4b1814e1957caa41a4b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 15 May 2023 11:34:05 +0800 Subject: [PATCH 02/28] feature: save work --- source/libs/scalar/src/filter.c | 57 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 77534db411..8c3e97ccbc 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3957,12 +3957,13 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // may be int64, uint64 and double + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) struct { - uint32_t nData; + uint32_t nData; // TODO maybe remove this and make pData len prefixed? uint8_t *pData; - }; // maybe varchar, nchar + }; // for varchar, nchar } datum; + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { @@ -3979,6 +3980,42 @@ typedef struct { } SFltSclRange; int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_NULL: { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 0 ; else return -1; + break; + } + case FLT_SCL_DATUM_KIND_MIN: { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 1; + else { + if (val2->kind == FLT_SCL_DATUM_KIND_MIN) return 0; else return -1; + } + break; + } + case FLT_SCL_DATUM_KIND_MAX: { + if (val2->kind == FLT_SCL_DATUM_KIND_MAX) return 0; else return 1; + } + case FLT_SCL_DATUM_KIND_UINT64: { + return compareUint64Val(&val1->datum.val, &val1->datum.val); + } + case FLT_SCL_DATUM_KIND_INT64: { + return compareInt64Val(&val1->datum.val, &val2->datum.val); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareFloatDouble(&val1->datum.val, &val2->datum.val); + } + case FLT_SCL_DATUM_KIND_NCHAR: { + return compareLenPrefixedWStr(&val1->datum.pData, &val2->datum.pData); //TODO + } + case FLT_SCL_DATUM_KIND_VARCHAR: { + return compareLenPrefixedStr(&val1->datum.pData, &val2->datum.pData); //TODO + } + default: + fltError("not supported kind. just return 0"); + return 0; + break; + } + return 0; } bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { @@ -4071,26 +4108,32 @@ typedef struct { SColumnNode* colNode; SValueNode* valNode; EOperatorType type; -} SFltSclScalarFunc; +} SFltSclScalarOp; +// simple define it, TODO: implement later typedef struct { SColumnNode* colNode; - } SFltSclConstColumn; +// simple define it, TODO: implement later typedef struct { SValueNode* value; } SFltSclConstant; + +int32_t fltSclBuildRangeFromScalarFunc(SFltSclScalarOp* func, SFltSclPoint* points) { + +} + + typedef struct { bool useRange; SHashObj* colRanges; - } SFltSclOptCtx; static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; - + return DEAL_RES_CONTINUE; } From e16f4c7c440ae02905e18b397eb985aa3b91f668 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 17 May 2023 16:57:09 +0800 Subject: [PATCH 03/28] fix: refactor and save work --- source/libs/scalar/src/filter.c | 220 ++++++++++++++++++-------------- 1 file changed, 124 insertions(+), 96 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8c3e97ccbc..85974bcd3a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -22,6 +22,7 @@ #include "sclInt.h" #include "tcompare.h" #include "tdatablock.h" +#include "tsimplehash.h" #include "ttime.h" bool filterRangeCompGi(const void *minv, const void *maxv, const void *minr, const void *maxr, __compar_fn_t cfunc) { @@ -260,7 +261,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { comparFn = 19; } else if (optr == OP_TYPE_NMATCH) { comparFn = 20; - } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ + } else if (optr == OP_TYPE_LIKE) { /* wildcard query using like operator */ comparFn = 7; } else if (optr == OP_TYPE_NOT_LIKE) { /* wildcard query using like operator */ comparFn = 26; @@ -1633,7 +1634,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SDataType *dType = &var->node.resType; qDebug("VAL%d => [type:%d][val:%" PRIx64 "]", i, dType->type, var->datum.i); // TODO } else if (field->data) { - qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO + qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO } } @@ -3957,59 +3958,96 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - struct { - uint32_t nData; // TODO maybe remove this and make pData len prefixed? - uint8_t *pData; - }; // for varchar, nchar + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) + uint8_t *pData; // for varchar, nchar } datum; - SDataType type; // TODO: original data type, may not be used? + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { SFltSclDatum val; - bool excl; - bool start; + bool excl; + bool start; } SFltSclPoint; typedef struct { SFltSclDatum low; SFltSclDatum high; - bool lowExcl; - bool highExcl; + bool lowExcl; + bool highExcl; } SFltSclRange; -int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 switch (val1->kind) { - case FLT_SCL_DATUM_KIND_NULL: { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 0 ; else return -1; - break; + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->datum.val, &val2->datum.val); } - case FLT_SCL_DATUM_KIND_MIN: { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL) return 1; - else { - if (val2->kind == FLT_SCL_DATUM_KIND_MIN) return 0; else return -1; - } - break; + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->datum.val, &val2->datum.val); } - case FLT_SCL_DATUM_KIND_MAX: { - if (val2->kind == FLT_SCL_DATUM_KIND_MAX) return 0; else return 1; + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->datum.val, &val2->datum.val); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->datum.val, &val2->datum.val); } + //TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + } + + switch (val2->kind) { case FLT_SCL_DATUM_KIND_UINT64: { - return compareUint64Val(&val1->datum.val, &val1->datum.val); + return fltSclCompareWithUInt64(val1, val2); } case FLT_SCL_DATUM_KIND_INT64: { - return compareInt64Val(&val1->datum.val, &val2->datum.val); + return fltSclCompareWithInt64(val1, val2); } case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareFloatDouble(&val1->datum.val, &val2->datum.val); - } - case FLT_SCL_DATUM_KIND_NCHAR: { - return compareLenPrefixedWStr(&val1->datum.pData, &val2->datum.pData); //TODO - } - case FLT_SCL_DATUM_KIND_VARCHAR: { - return compareLenPrefixedStr(&val1->datum.pData, &val2->datum.pData); //TODO + return fltSclCompareWithFloat64(val1, val2); } + //TODO: varchar/nchar default: fltError("not supported kind. just return 0"); return 0; @@ -4018,125 +4056,115 @@ int32_t fltSclCompareDatum(SFltSclDatum* val1, SFltSclDatum* val2) { return 0; } -bool fltSclLessPoint(SFltSclPoint* pt1, SFltSclPoint* pt2) { +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { // first value compare int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); if (cmp != 0) { - return cmp < 0 ; + return cmp < 0; } if (pt1->start && pt2->start) { - return !pt1->excl && pt2->excl; - } else if (pt1->start) { - return pt1->excl && !pt2->excl; - } else if (pt2->start) { - return pt1->excl || pt2->excl; - } + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } return pt1->excl && !pt2->excl; } -int32_t fltSclMergeSort(SArray* pts1, SArray* pts2, SArray* result) { +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); size_t i = 0; size_t j = 0; while (i < len1 && j < len2) { - SFltSclPoint* pt1 = taosArrayGet(pts1, i); - SFltSclPoint* pt2 = taosArrayGet(pts2, j); - bool less = fltSclLessPoint(pt1, pt2); - if (less) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { taosArrayPush(result, pt1); ++i; - } else { + } else { taosArrayPush(result, pt2); ++j; - } + } } if (i < len1) { - for (; i < len1; ++i) { - SFltSclPoint* pt1 = taosArrayGet(pts1, i); + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); taosArrayPush(result, pt1); - } + } } if (j < len2) { - for (; j < len2; ++j) { + for (; j < len2; ++j) { SFltSclPoint *pt2 = taosArrayGet(pts2, j); taosArrayPush(result, pt2); - } + } } return 0; } // pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range -int32_t fltSclMerge(SArray* pts1, SArray* pts2, bool isUnion, SArray* merged) { +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); - //first merge sort pts1 and pts2 - SArray* all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); fltSclMergeSort(pts1, pts2, all); int32_t countRequired = (isUnion) ? 1 : 2; int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint* pt = taosArrayGet(pts1, i); - if (pt->start) { - ++count; - if (count == countRequired) { - taosArrayPush(merged, pt); - } - } else { - if (count == countRequired) { - taosArrayPush(merged, pt); - } - --count; + SFltSclPoint *pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } } taosArrayDestroy(all); return 0; } +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } -int32_t fltSclIntersect(SArray* pts1, SArray* pts2, SArray* merged) { - return fltSclMerge(pts1, pts2, false, merged); -} - -int32_t fltSclUnion(SArray* pts1, SArray* pts2, SArray* merged) { - return fltSclMerge(pts1, pts2, true, merged); -} - +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } typedef struct { - SColumnNode* colNode; - SValueNode* valNode; + SColumnNode *colNode; + SValueNode *valNode; EOperatorType type; -} SFltSclScalarOp; +} SFltSclOperator; -// simple define it, TODO: implement later -typedef struct { - SColumnNode* colNode; -} SFltSclConstColumn; - -// simple define it, TODO: implement later -typedef struct { - SValueNode* value; -} SFltSclConstant; - - -int32_t fltSclBuildRangeFromScalarFunc(SFltSclScalarOp* func, SFltSclPoint* points) { +//TODO: column, constant, operator, and/or/not +int32_t fltSclBuildRangeFromOperator(SFltSclOperator *sclOper, SSHashObj* obj) { + switch (sclOper->type) { + case OP_TYPE_GREATER_THAN: { + } + break; + default: + break; + } + return TSDB_CODE_SUCCESS; } - typedef struct { - bool useRange; - SHashObj* colRanges; + bool useRange; + SHashObj *colRanges; } SFltSclOptCtx; -static EDealRes fltSclMayBeOptimed(SNode* pNode, void* pCtx) { - SFltSclOptCtx* ctx = (SFltSclOptCtx*)pCtx; +static EDealRes fltSclMayBeOptimed(SNode *pNode, void *pCtx) { + SFltSclOptCtx *ctx = (SFltSclOptCtx *)pCtx; return DEAL_RES_CONTINUE; } - int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { SFltSclOptCtx ctx; nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); From 09837b7639190340f0d9a339a6da7db163b56ddd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 18 May 2023 19:00:10 +0800 Subject: [PATCH 04/28] fix: save work --- source/libs/scalar/src/filter.c | 124 ++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 15 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 85974bcd3a..01bd53008e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3960,7 +3960,7 @@ typedef struct { union { int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) uint8_t *pData; // for varchar, nchar - } datum; + }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3981,11 +3981,11 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2->kind == int64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Double(&val1->datum.val, &val2->datum.val); + return compareUint64Double(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Double(&val1->datum.val, &val2->datum.val); + return compareInt64Double(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleVal(&val1->datum.val, &val2->datum.val); + return compareDoubleVal(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -3998,11 +3998,11 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2->kind == int64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Int64(&val1->datum.val, &val2->datum.val); + return compareUint64Int64(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Val(&val1->datum.val, &val2->datum.val); + return compareInt64Val(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleInt64(&val1->datum.val, &val2->datum.val); + return compareDoubleInt64(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -4015,11 +4015,11 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { // val2 kind == uint64 switch (val1->kind) { case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Val(&val1->datum.val, &val2->datum.val); + return compareUint64Val(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Uint64(&val1->datum.val, &val2->datum.val); + return compareInt64Uint64(&val1->val, &val2->val); case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleUint64(&val1->datum.val, &val2->datum.val); + return compareDoubleUint64(&val1->val, &val2->val); } //TODO: varchar, nchar default: @@ -4142,15 +4142,109 @@ typedef struct { EOperatorType type; } SFltSclOperator; -//TODO: column, constant, operator, and/or/not +//TODO: column, constant -int32_t fltSclBuildRangeFromOperator(SFltSclOperator *sclOper, SSHashObj* obj) { - switch (sclOper->type) { - case OP_TYPE_GREATER_THAN: { +typedef struct { + SColumnNode* colNode; + SArray* points; +} SFltSclColumnRange; + +SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode*)colRange->colNode, (SNode*)colNode)) { + return colRange; } + } + SFltSclColumnRange newColRange = {.colNode = colNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->val = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->val = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + *(uint64_t *)&datum->val = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + *(double *)&datum->val = valNode->datum.d; + break; + } + //TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { + switch (oper->type) { + case OP_TYPE_GREATER_THAN: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &start); + taosArrayPush(points, &end); break; - default: + } + default: { + qError("not supported op"); break; + } + } + return TSDB_CODE_SUCCESS; +} + +//TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray* sclOpList, SArray* colRangeList) { + size_t sz = taosArrayGetSize(sclOpList); + for (int32_t i = 0; i < sz; ++i) { + SFltSclOperator* sclOper = taosArrayGet(sclOpList, i); + SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); + SArray* points = taosArrayInit(4, sizeof(SFltSclPoint)); + fltSclBuildRangePoints(sclOper, points); + SArray* merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } return TSDB_CODE_SUCCESS; } From 3e92ec7a8a4c6f701c31a95a2c4d80e816e6817f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 14:09:37 +0800 Subject: [PATCH 05/28] fix: save work --- source/libs/scalar/src/filter.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 01bd53008e..ffc165cba9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4223,6 +4223,16 @@ int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { taosArrayPush(points, &end); break; } + case OP_TYPE_GREATER_EQUAL: { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &start); + taosArrayPush(points, &end); + } + case default: { qError("not supported op"); break; From 1f6d894dfd12170afd9208e67fb0136faf30b420 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 19 May 2023 17:06:03 +0800 Subject: [PATCH 06/28] fix: save work --- source/libs/scalar/src/filter.c | 121 ++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 30 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ffc165cba9..fd6e79ed42 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3958,10 +3958,10 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - uint8_t *pData; // for varchar, nchar + int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) + uint8_t *pData; // for varchar, nchar }; - SDataType type; // TODO: original data type, may not be used? + SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; typedef struct { @@ -3987,7 +3987,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleVal(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4004,7 +4004,7 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleInt64(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4021,7 +4021,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return compareDoubleUint64(&val1->val, &val2->val); } - //TODO: varchar, nchar + // TODO: varchar, nchar default: qError("not support comparsion. %d, %d", val1->kind, val2->kind); return (val1->kind - val2->kind); @@ -4047,7 +4047,7 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { case FLT_SCL_DATUM_KIND_FLOAT64: { return fltSclCompareWithFloat64(val1, val2); } - //TODO: varchar/nchar + // TODO: varchar/nchar default: fltError("not supported kind. just return 0"); return 0; @@ -4142,17 +4142,17 @@ typedef struct { EOperatorType type; } SFltSclOperator; -//TODO: column, constant +// TODO: column, constant typedef struct { - SColumnNode* colNode; - SArray* points; + SColumnNode *colNode; + SArray *points; } SFltSclColumnRange; -SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeList) { +SFltSclColumnRange *fltSclGetColumnRange(SColumnNode *colNode, SArray *colRangeList) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(colRangeList, i); - if (nodesEqualNode((SNode*)colRange->colNode, (SNode*)colNode)) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { return colRange; } } @@ -4161,7 +4161,7 @@ SFltSclColumnRange* fltSclGetColumnRange(SColumnNode* colNode, SArray* colRangeL return taosArrayGetLast(colRangeList); } -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) { +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { datum->type = valNode->node.resType; if (valNode->isNull) { @@ -4200,39 +4200,101 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum* datum, SValueNode* valNode) *(double *)&datum->val = valNode->datum.d; break; } - //TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { qError("not supported"); break; } } - } return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { +int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { + if (IS_UNSIGNED_NUMERIC_TYPE(oper->colNode->node.resType.type) && + ((IS_SIGNED_NUMERIC_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.i < 0) || + (IS_FLOAT_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.d < 0))) { + if (oper->type == OP_TYPE_GREATER_THAN || oper->type == OP_TYPE_GREATER_EQUAL || oper->type == OP_TYPE_NOT_EQUAL) { + SFltSclDatum start; + fltSclBuildDatumFromValueNode(&start, oper->valNode); + start.val = 0; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + return TSDB_CODE_SUCCESS; + } switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; fltSclBuildDatumFromValueNode(&start, oper->valNode); SFltSclPoint startPt = {.start = true, .excl = true, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &start); - taosArrayPush(points, &end); + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); break; } case OP_TYPE_GREATER_EQUAL: { SFltSclDatum start; fltSclBuildDatumFromValueNode(&start, oper->valNode); SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &start); - taosArrayPush(points, &end); + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_THAN: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = true, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_LOWER_EQUAL: { + SFltSclDatum end; + fltSclBuildDatumFromValueNode(&end, oper->valNode); + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + SFltSclPoint startPt = {.start = true, .excl = false, .val = valDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_NOT_EQUAL: { + SFltSclDatum valDatum; + fltSclBuildDatumFromValueNode(&valDatum, oper->valNode); + { + SFltSclDatum start = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; + SFltSclPoint endPt = {.start = false, .excl = true, .val = valDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + { + SFltSclPoint startPt = {.start = true, .excl = true, .val = valDatum}; + SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } + break; } - case default: { qError("not supported op"); break; @@ -4241,20 +4303,19 @@ int32_t fltSclBuildRangePoints(SFltSclOperator* oper, SArray* points) { return TSDB_CODE_SUCCESS; } -//TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray* sclOpList, SArray* colRangeList) { +// TODO: process DNF composed of CNF +int32_t fltSclProcessCNF(SArray *sclOpList, SArray *colRangeList) { size_t sz = taosArrayGetSize(sclOpList); for (int32_t i = 0; i < sz; ++i) { - SFltSclOperator* sclOper = taosArrayGet(sclOpList, i); + SFltSclOperator *sclOper = taosArrayGet(sclOpList, i); SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); - SArray* points = taosArrayInit(4, sizeof(SFltSclPoint)); + SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); fltSclBuildRangePoints(sclOper, points); - SArray* merged = taosArrayInit(4, sizeof(SFltSclPoint)); + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); int32_t code = fltSclIntersect(colRange->points, points, merged); taosArrayDestroy(colRange->points); taosArrayDestroy(points); colRange->points = merged; - } return TSDB_CODE_SUCCESS; } From 6456d5dccfd11afe518fc3a282494eb50a0e8a38 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 22 May 2023 11:41:19 +0800 Subject: [PATCH 07/28] enhance: code finish --- source/libs/scalar/inc/filterInt.h | 7 + source/libs/scalar/src/filter.c | 700 ++++++++++++++++------------- 2 files changed, 407 insertions(+), 300 deletions(-) diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 2023d38777..1ca8ac1d8c 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -227,8 +227,10 @@ typedef struct SFltTreeStat { SFilterInfo *info; } SFltTreeStat; + typedef struct SFltScalarCtx { SNode *node; + SArray* fltSclRange; } SFltScalarCtx; typedef struct SFltBuildGroupCtx { @@ -237,6 +239,11 @@ typedef struct SFltBuildGroupCtx { int32_t code; } SFltBuildGroupCtx; +typedef struct { + SColumnNode *colNode; + SArray *points; +} SFltSclColumnRange; + struct SFilterInfo { bool scalarMode; SFltScalarCtx sclCtx; diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index fd6e79ed42..33f37c722e 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1841,6 +1841,13 @@ void filterFreeInfo(SFilterInfo *info) { return; } + for (int32_t i = 0 ; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode*)colRange->colNode); + taosArrayDestroy(colRange->points); + } + taosArrayDestroy(info->sclCtx.fltSclRange); + taosMemoryFreeClear(info->cunits); taosMemoryFreeClear(info->blkUnitRes); taosMemoryFreeClear(info->blkUnits); @@ -3424,8 +3431,335 @@ _return: return code; } + +// compare ranges, null < min < val < max. null=null, min=min, max=max +typedef enum { + FLT_SCL_DATUM_KIND_NULL, + FLT_SCL_DATUM_KIND_MIN, + FLT_SCL_DATUM_KIND_INT64, + FLT_SCL_DATUM_KIND_UINT64, + FLT_SCL_DATUM_KIND_FLOAT64, + FLT_SCL_DATUM_KIND_VARCHAR, + FLT_SCL_DATUM_KIND_NCHAR, + FLT_SCL_DATUM_KIND_MAX, +} SFltSclDatumKind; + +typedef struct { + SFltSclDatumKind kind; + union { + int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) + uint64_t u; + double d; + uint8_t *pData; // for varchar, nchar + }; + SDataType type; // TODO: original data type, may not be used? +} SFltSclDatum; + +typedef struct { + SFltSclDatum val; + bool excl; + bool start; +} SFltSclPoint; + +int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == float64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Double(&val1->u, &val2->d); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Double(&val1->i, &val2->d); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleVal(&val1->d, &val2->d); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2->kind == int64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Int64(&val1->u, &val2->i); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Val(&val1->i, &val2->i); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleInt64(&val1->d, &val2->i); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { + // val2 kind == uint64 + switch (val1->kind) { + case FLT_SCL_DATUM_KIND_UINT64: + return compareUint64Val(&val1->u, &val2->u); + case FLT_SCL_DATUM_KIND_INT64: + return compareInt64Uint64(&val1->i, &val2->u); + case FLT_SCL_DATUM_KIND_FLOAT64: { + return compareDoubleUint64(&val1->d, &val2->u); + } + // TODO: varchar, nchar + default: + qError("not support comparsion. %d, %d", val1->kind, val2->kind); + return (val1->kind - val2->kind); + } +} + +int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { + if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || + val2->kind == FLT_SCL_DATUM_KIND_MAX) { + if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || + val1->kind == FLT_SCL_DATUM_KIND_MAX) { + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + } + } + + switch (val2->kind) { + case FLT_SCL_DATUM_KIND_UINT64: { + return fltSclCompareWithUInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_INT64: { + return fltSclCompareWithInt64(val1, val2); + } + case FLT_SCL_DATUM_KIND_FLOAT64: { + return fltSclCompareWithFloat64(val1, val2); + } + // TODO: varchar/nchar + default: + fltError("not supported kind. just return 0"); + return 0; + break; + } + return 0; +} + +bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { + // first value compare + int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); + if (cmp != 0) { + return cmp < 0; + } + + if (pt1->start && pt2->start) { + return !pt1->excl && pt2->excl; + } else if (pt1->start) { + return pt1->excl && !pt2->excl; + } else if (pt2->start) { + return pt1->excl || pt2->excl; + } + return pt1->excl && !pt2->excl; +} + +int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + size_t i = 0; + size_t j = 0; + while (i < len1 && j < len2) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + bool less = fltSclLessPoint(pt1, pt2); + if (less) { + taosArrayPush(result, pt1); + ++i; + } else { + taosArrayPush(result, pt2); + ++j; + } + } + if (i < len1) { + for (; i < len1; ++i) { + SFltSclPoint *pt1 = taosArrayGet(pts1, i); + taosArrayPush(result, pt1); + } + } + if (j < len2) { + for (; j < len2; ++j) { + SFltSclPoint *pt2 = taosArrayGet(pts2, j); + taosArrayPush(result, pt2); + } + } + return 0; +} + +// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range +int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { + size_t len1 = taosArrayGetSize(pts1); + size_t len2 = taosArrayGetSize(pts2); + // first merge sort pts1 and pts2 + SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); + fltSclMergeSort(pts1, pts2, all); + int32_t countRequired = (isUnion) ? 1 : 2; + int32_t count = 0; + for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { + SFltSclPoint *pt = taosArrayGet(pts1, i); + if (pt->start) { + ++count; + if (count == countRequired) { + taosArrayPush(merged, pt); + } + } else { + if (count == countRequired) { + taosArrayPush(merged, pt); + } + --count; + } + } + taosArrayDestroy(all); + return 0; +} + +int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } + +int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } + +// TODO: column, constant +typedef struct { + SColumnNode *colNode; + SValueNode *valNode; + EOperatorType type; +} SFltSclOperator; + + +SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { + for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); + if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { + return colRange; + } + } + SColumnNode* pColumnNode = (SColumnNode*)nodesCloneNode((SNode*)colNode); + SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; + taosArrayPush(colRangeList, &newColRange); + return taosArrayGetLast(colRangeList); +} + +int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { + datum->type = valNode->node.resType; + + if (valNode->isNull) { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + } else { + switch (valNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: { + datum->kind = FLT_SCL_DATUM_KIND_NULL; + break; + } + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = (valNode->datum.b) ? 0 : 1; + break; + } + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = valNode->datum.i; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = valNode->datum.u; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = valNode->datum.d; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int64_t val) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: { + datum->kind = FLT_SCL_DATUM_KIND_INT64; + datum->i = val; + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: { + datum->kind = FLT_SCL_DATUM_KIND_UINT64; + datum->u = *(uint64_t*)&val; + break; + } + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: { + datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; + datum->d = *(double*)&val; + break; + } + // TODO:varchar/nchar/json + default: { + qError("not supported"); + break; + } + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, SArray* points) { + SFltSclDatum min; + fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); + SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; + SFltSclDatum max; + fltSclBuildDatumFromBlockSmaValue(&max, colRange->colNode->node.resType.type, pAgg->max); + SFltSclPoint maxPt = {.excl = false, .start = false, .val = max}; + taosArrayPush(points, &minPt); + taosArrayPush(points, &maxPt); + return TSDB_CODE_SUCCESS; +} + bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { + SArray* colRanges = info->sclCtx.fltSclRange; + for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { + SFltSclColumnRange* colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; + for (; j < numOfCols; ++j) { + if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { + foundCol = true; + } + } + if (foundCol) { + SColumnDataAgg* pAgg = pDataStatis[j]; + SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); + fltSclBuildRangeFromBlockSma(colRange, pAgg, points); + SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); + fltSclIntersect(points, colRange->points, merged); + bool isIntersect = taosArrayGetSize(merged) != 0; + taosArrayDestroy(merged); + taosArrayDestroy(points); + if (!isIntersect) { + return false; + } + } + } return true; } @@ -3943,289 +4277,7 @@ _return: FLT_RET(code); } -// compare ranges, null < min < val < max. null=null, min=min, max=max -typedef enum { - FLT_SCL_DATUM_KIND_NULL, - FLT_SCL_DATUM_KIND_MIN, - FLT_SCL_DATUM_KIND_INT64, - FLT_SCL_DATUM_KIND_UINT64, - FLT_SCL_DATUM_KIND_FLOAT64, - FLT_SCL_DATUM_KIND_VARCHAR, - FLT_SCL_DATUM_KIND_NCHAR, - FLT_SCL_DATUM_KIND_MAX, -} SFltSclDatumKind; - -typedef struct { - SFltSclDatumKind kind; - union { - int64_t val; // for int64, uint64 and double and bool (1 true, 0 false) - uint8_t *pData; // for varchar, nchar - }; - SDataType type; // TODO: original data type, may not be used? -} SFltSclDatum; - -typedef struct { - SFltSclDatum val; - bool excl; - bool start; -} SFltSclPoint; - -typedef struct { - SFltSclDatum low; - SFltSclDatum high; - bool lowExcl; - bool highExcl; -} SFltSclRange; - -int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2->kind == int64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Double(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Double(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleVal(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2->kind == int64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Int64(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Val(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleInt64(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { - // val2 kind == uint64 - switch (val1->kind) { - case FLT_SCL_DATUM_KIND_UINT64: - return compareUint64Val(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_INT64: - return compareInt64Uint64(&val1->val, &val2->val); - case FLT_SCL_DATUM_KIND_FLOAT64: { - return compareDoubleUint64(&val1->val, &val2->val); - } - // TODO: varchar, nchar - default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); - return (val1->kind - val2->kind); - } -} - -int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { - if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || - val2->kind == FLT_SCL_DATUM_KIND_MAX) { - if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || - val1->kind == FLT_SCL_DATUM_KIND_MAX) { - return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); - } - } - - switch (val2->kind) { - case FLT_SCL_DATUM_KIND_UINT64: { - return fltSclCompareWithUInt64(val1, val2); - } - case FLT_SCL_DATUM_KIND_INT64: { - return fltSclCompareWithInt64(val1, val2); - } - case FLT_SCL_DATUM_KIND_FLOAT64: { - return fltSclCompareWithFloat64(val1, val2); - } - // TODO: varchar/nchar - default: - fltError("not supported kind. just return 0"); - return 0; - break; - } - return 0; -} - -bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { - // first value compare - int32_t cmp = fltSclCompareDatum(&pt1->val, &pt2->val); - if (cmp != 0) { - return cmp < 0; - } - - if (pt1->start && pt2->start) { - return !pt1->excl && pt2->excl; - } else if (pt1->start) { - return pt1->excl && !pt2->excl; - } else if (pt2->start) { - return pt1->excl || pt2->excl; - } - return pt1->excl && !pt2->excl; -} - -int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { - size_t len1 = taosArrayGetSize(pts1); - size_t len2 = taosArrayGetSize(pts2); - size_t i = 0; - size_t j = 0; - while (i < len1 && j < len2) { - SFltSclPoint *pt1 = taosArrayGet(pts1, i); - SFltSclPoint *pt2 = taosArrayGet(pts2, j); - bool less = fltSclLessPoint(pt1, pt2); - if (less) { - taosArrayPush(result, pt1); - ++i; - } else { - taosArrayPush(result, pt2); - ++j; - } - } - if (i < len1) { - for (; i < len1; ++i) { - SFltSclPoint *pt1 = taosArrayGet(pts1, i); - taosArrayPush(result, pt1); - } - } - if (j < len2) { - for (; j < len2; ++j) { - SFltSclPoint *pt2 = taosArrayGet(pts2, j); - taosArrayPush(result, pt2); - } - } - return 0; -} - -// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range -int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { - size_t len1 = taosArrayGetSize(pts1); - size_t len2 = taosArrayGetSize(pts2); - // first merge sort pts1 and pts2 - SArray *all = taosArrayInit(len1 + len2, sizeof(SFltSclPoint)); - fltSclMergeSort(pts1, pts2, all); - int32_t countRequired = (isUnion) ? 1 : 2; - int32_t count = 0; - for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint *pt = taosArrayGet(pts1, i); - if (pt->start) { - ++count; - if (count == countRequired) { - taosArrayPush(merged, pt); - } - } else { - if (count == countRequired) { - taosArrayPush(merged, pt); - } - --count; - } - } - taosArrayDestroy(all); - return 0; -} - -int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, false, merged); } - -int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } - -typedef struct { - SColumnNode *colNode; - SValueNode *valNode; - EOperatorType type; -} SFltSclOperator; - -// TODO: column, constant - -typedef struct { - SColumnNode *colNode; - SArray *points; -} SFltSclColumnRange; - -SFltSclColumnRange *fltSclGetColumnRange(SColumnNode *colNode, SArray *colRangeList) { - for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { - SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); - if (nodesEqualNode((SNode *)colRange->colNode, (SNode *)colNode)) { - return colRange; - } - } - SFltSclColumnRange newColRange = {.colNode = colNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; - taosArrayPush(colRangeList, &newColRange); - return taosArrayGetLast(colRangeList); -} - -int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) { - datum->type = valNode->node.resType; - - if (valNode->isNull) { - datum->kind = FLT_SCL_DATUM_KIND_NULL; - } else { - switch (valNode->node.resType.type) { - case TSDB_DATA_TYPE_NULL: { - datum->kind = FLT_SCL_DATUM_KIND_NULL; - break; - } - case TSDB_DATA_TYPE_BOOL: { - datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->val = (valNode->datum.b) ? 0 : 1; - break; - } - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: { - datum->kind = FLT_SCL_DATUM_KIND_INT64; - datum->val = valNode->datum.i; - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: { - datum->kind = FLT_SCL_DATUM_KIND_UINT64; - *(uint64_t *)&datum->val = valNode->datum.u; - break; - } - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - *(double *)&datum->val = valNode->datum.d; - break; - } - // TODO:varchar/nchar/json - default: { - qError("not supported"); - break; - } - } - } - return TSDB_CODE_SUCCESS; -} - int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { - if (IS_UNSIGNED_NUMERIC_TYPE(oper->colNode->node.resType.type) && - ((IS_SIGNED_NUMERIC_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.i < 0) || - (IS_FLOAT_TYPE(oper->valNode->node.resType.type) && oper->valNode->datum.d < 0))) { - if (oper->type == OP_TYPE_GREATER_THAN || oper->type == OP_TYPE_GREATER_EQUAL || oper->type == OP_TYPE_NOT_EQUAL) { - SFltSclDatum start; - fltSclBuildDatumFromValueNode(&start, oper->valNode); - start.val = 0; - SFltSclPoint startPt = {.start = true, .excl = false, .val = start}; - SFltSclDatum end = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; - SFltSclPoint endPt = {.start = false, .excl = false, .val = end}; - taosArrayPush(points, &startPt); - taosArrayPush(points, &endPt); - } - return TSDB_CODE_SUCCESS; - } switch (oper->type) { case OP_TYPE_GREATER_THAN: { SFltSclDatum start; @@ -4304,36 +4356,84 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } // TODO: process DNF composed of CNF -int32_t fltSclProcessCNF(SArray *sclOpList, SArray *colRangeList) { - size_t sz = taosArrayGetSize(sclOpList); +int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { + size_t sz = taosArrayGetSize(sclOpListCNF); for (int32_t i = 0; i < sz; ++i) { - SFltSclOperator *sclOper = taosArrayGet(sclOpList, i); - SFltSclColumnRange *colRange = fltSclGetColumnRange(sclOper->colNode, colRangeList); + SFltSclOperator *sclOper = taosArrayGet(sclOpListCNF, i); + SFltSclColumnRange *colRange = fltSclGetOrCreateColumnRange(sclOper->colNode, colRangeList); SArray *points = taosArrayInit(4, sizeof(SFltSclPoint)); fltSclBuildRangePoints(sclOper, points); - SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); - int32_t code = fltSclIntersect(colRange->points, points, merged); - taosArrayDestroy(colRange->points); - taosArrayDestroy(points); - colRange->points = merged; + if (taosArrayGetSize(colRange->points) != 0) { + SArray *merged = taosArrayInit(4, sizeof(SFltSclPoint)); + int32_t code = fltSclIntersect(colRange->points, points, merged); + taosArrayDestroy(colRange->points); + taosArrayDestroy(points); + colRange->points = merged; + } else { + colRange->points = points; + } + } return TSDB_CODE_SUCCESS; } -typedef struct { - bool useRange; - SHashObj *colRanges; -} SFltSclOptCtx; +static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { + if (nodeType(pNode) != QUERY_NODE_OPERATOR) { + return TSDB_CODE_SUCCESS; + } + SOperatorNode* pOper = (SOperatorNode*)pNode; + //TODO: left value node, right column node + //TODO: datatype + //TODO: operator + if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && + (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + SValueNode* valNode = (SValueNode*)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SNode *p = nodesCloneNode(pNode); + taosArrayPush(sclOpList, &p); + } + } + return TSDB_CODE_SUCCESS; +} -static EDealRes fltSclMayBeOptimed(SNode *pNode, void *pCtx) { - SFltSclOptCtx *ctx = (SFltSclOptCtx *)pCtx; - return DEAL_RES_CONTINUE; +static int32_t fltSclCollectOperatorsFromLogicCond(SNode* pNode, SArray* sclOpList) { + if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { + return TSDB_CODE_SUCCESS; + } + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pNode; + //TODO: support LOGIC_COND_TYPE_OR + if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { + return TSDB_CODE_SUCCESS; + } + SNode* pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { + if (nodeType(pNode) == QUERY_NODE_OPERATOR) { + fltSclCollectOperatorFromNode(pNode, sclOpList); + } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { + fltSclCollectOperatorsFromLogicCond(pNode, sclOpList); + } + return TSDB_CODE_SUCCESS; } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SFltSclOptCtx ctx; - nodesWalkExprPostOrder(*pNode, fltSclMayBeOptimed, &ctx); + SArray* sclOpList = taosArrayInit(16, POINTER_BYTES); + fltSclCollectOperators(*pNode, sclOpList); + SArray* colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + fltSclProcessCNF(sclOpList, colRangeList); + pInfo->sclCtx.fltSclRange = colRangeList; + for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { + SNode* p = taosArrayGetP(sclOpList, i); + nodesDestroyNode(p); + } + taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; } From 43ac6fb1fd532c96f5deba1b21e46abd5b2e0762 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 23 May 2023 08:43:31 +0800 Subject: [PATCH 08/28] fix: all null from block sma --- source/libs/scalar/src/filter.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 33f37c722e..4475a07dcd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3722,7 +3722,15 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, SArray* points) { +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, int32_t numOfRows, SArray* points) { + if (pAgg->numOfNull == numOfRows) { + SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = datum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + return TSDB_CODE_SUCCESS; + } SFltSclDatum min; fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; @@ -3749,7 +3757,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t if (foundCol) { SColumnDataAgg* pAgg = pDataStatis[j]; SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); - fltSclBuildRangeFromBlockSma(colRange, pAgg, points); + fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; From 24ae8d7ecb870852c9979083eddb5686b32bb278 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 09:49:42 +0800 Subject: [PATCH 09/28] fix: fix ci memory leak error --- source/libs/scalar/src/filter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4475a07dcd..c21abc1380 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4378,6 +4378,7 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(points); colRange->points = merged; } else { + taosArrayDestroy(colRange->points); colRange->points = points; } From 77a1de444c24bd68d0db36c2d7820feb4b1a8c4b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 10:47:58 +0800 Subject: [PATCH 10/28] fix: where c7 means c7 is true, the right child is null --- source/libs/scalar/src/filter.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c21abc1380..4a9ebb16c1 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4394,6 +4394,9 @@ static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { //TODO: left value node, right column node //TODO: datatype //TODO: operator + if (pOper->pLeft == NULL || pOper->pRight == NULL) { + return TSDB_CODE_SUCCESS; + } if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || From c9140e547e652e7e241abeee7b5fc175bcbeeb4c Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 23 May 2023 14:08:20 +0800 Subject: [PATCH 11/28] fix: fix bugs of wrongly use sfltscloperator --- source/libs/scalar/src/filter.c | 76 ++++++++++++++++----------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4a9ebb16c1..3b61e3a97a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1841,9 +1841,9 @@ void filterFreeInfo(SFilterInfo *info) { return; } - for (int32_t i = 0 ; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(info->sclCtx.fltSclRange, i); - nodesDestroyNode((SNode*)colRange->colNode); + for (int32_t i = 0; i < taosArrayGetSize(info->sclCtx.fltSclRange); ++i) { + SFltSclColumnRange *colRange = taosArrayGet(info->sclCtx.fltSclRange, i); + nodesDestroyNode((SNode *)colRange->colNode); taosArrayDestroy(colRange->points); } taosArrayDestroy(info->sclCtx.fltSclRange); @@ -3431,7 +3431,6 @@ _return: return code; } - // compare ranges, null < min < val < max. null=null, min=min, max=max typedef enum { FLT_SCL_DATUM_KIND_NULL, @@ -3447,7 +3446,7 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) + int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) uint64_t u; double d; uint8_t *pData; // for varchar, nchar @@ -3627,7 +3626,6 @@ typedef struct { EOperatorType type; } SFltSclOperator; - SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *colRangeList) { for (int32_t i = 0; i < taosArrayGetSize(colRangeList); ++i) { SFltSclColumnRange *colRange = taosArrayGet(colRangeList, i); @@ -3635,7 +3633,7 @@ SFltSclColumnRange *fltSclGetOrCreateColumnRange(SColumnNode *colNode, SArray *c return colRange; } } - SColumnNode* pColumnNode = (SColumnNode*)nodesCloneNode((SNode*)colNode); + SColumnNode *pColumnNode = (SColumnNode *)nodesCloneNode((SNode *)colNode); SFltSclColumnRange newColRange = {.colNode = pColumnNode, .points = taosArrayInit(4, sizeof(SFltSclPoint))}; taosArrayPush(colRangeList, &newColRange); return taosArrayGetLast(colRangeList); @@ -3690,7 +3688,7 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int64_t val) { +int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { case TSDB_DATA_TYPE_BOOL: { datum->kind = FLT_SCL_DATUM_KIND_INT64; @@ -3702,13 +3700,13 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: { datum->kind = FLT_SCL_DATUM_KIND_UINT64; - datum->u = *(uint64_t*)&val; + datum->u = *(uint64_t *)&val; break; } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { datum->kind = FLT_SCL_DATUM_KIND_FLOAT64; - datum->d = *(double*)&val; + datum->d = *(double *)&val; break; } // TODO:varchar/nchar/json @@ -3718,11 +3716,11 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum* datum, uint8_t type, int } } - return TSDB_CODE_SUCCESS; } -int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAgg* pAgg, int32_t numOfRows, SArray* points) { +int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAgg *pAgg, int32_t numOfRows, + SArray *points) { if (pAgg->numOfNull == numOfRows) { SFltSclDatum datum = {.kind = FLT_SCL_DATUM_KIND_NULL}; SFltSclPoint startPt = {.start = true, .excl = false, .val = datum}; @@ -3744,21 +3742,21 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange* colRange, SColumnDataAg bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { - SArray* colRanges = info->sclCtx.fltSclRange; + SArray *colRanges = info->sclCtx.fltSclRange; for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { - SFltSclColumnRange* colRange = taosArrayGet(colRanges, i); - bool foundCol = false; - int32_t j = 0; + SFltSclColumnRange *colRange = taosArrayGet(colRanges, i); + bool foundCol = false; + int32_t j = 0; for (; j < numOfCols; ++j) { if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { foundCol = true; } } if (foundCol) { - SColumnDataAgg* pAgg = pDataStatis[j]; - SArray* points = taosArrayInit(2, sizeof(SFltSclPoint)); + SColumnDataAgg *pAgg = pDataStatis[j]; + SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - SArray* merged = taosArrayInit(8, sizeof(SFltSclPoint)); + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; taosArrayDestroy(merged); @@ -4381,19 +4379,18 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { taosArrayDestroy(colRange->points); colRange->points = points; } - } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { return TSDB_CODE_SUCCESS; } - SOperatorNode* pOper = (SOperatorNode*)pNode; - //TODO: left value node, right column node - //TODO: datatype - //TODO: operator + SOperatorNode *pOper = (SOperatorNode *)pNode; + // TODO: left value node, right column node + // TODO: datatype + // TODO: operator if (pOper->pLeft == NULL || pOper->pRight == NULL) { return TSDB_CODE_SUCCESS; } @@ -4401,31 +4398,31 @@ static int32_t fltSclCollectOperatorFromNode(SNode* pNode, SArray* sclOpList) { (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - SValueNode* valNode = (SValueNode*)pOper->pRight; + SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - SNode *p = nodesCloneNode(pNode); - taosArrayPush(sclOpList, &p); + SFltSclOperator sclOp = { + .colNode = nodesCloneNode(pOper->pLeft), .valNode = nodesCloneNode(pOper->pRight), .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); } } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorsFromLogicCond(SNode* pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_LOGIC_CONDITION) { return TSDB_CODE_SUCCESS; } - SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pNode; - //TODO: support LOGIC_COND_TYPE_OR + SLogicConditionNode *pLogicCond = (SLogicConditionNode *)pNode; + // TODO: support LOGIC_COND_TYPE_OR if (pLogicCond->condType != LOGIC_COND_TYPE_AND) { return TSDB_CODE_SUCCESS; } - SNode* pExpr = NULL; - FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); - } + SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { +static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) == QUERY_NODE_OPERATOR) { fltSclCollectOperatorFromNode(pNode, sclOpList); } else if (nodeType(pNode) == QUERY_NODE_LOGIC_CONDITION) { @@ -4435,15 +4432,16 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray* sclOpList) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SArray* sclOpList = taosArrayInit(16, POINTER_BYTES); + SArray *sclOpList = taosArrayInit(16, POINTER_BYTES); fltSclCollectOperators(*pNode, sclOpList); - SArray* colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); + SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); fltSclProcessCNF(sclOpList, colRangeList); pInfo->sclCtx.fltSclRange = colRangeList; for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { - SNode* p = taosArrayGetP(sclOpList, i); - nodesDestroyNode(p); + SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); + nodesDestroyNode(sclOp->colNode); + nodesDestroyNode(sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 99620d434da9fdf995a923380416e5138ce5a32a Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 14:11:52 +0800 Subject: [PATCH 12/28] fix: fix bugs --- source/libs/scalar/src/filter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3b61e3a97a..c1de62a407 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4401,7 +4401,7 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = { - .colNode = nodesCloneNode(pOper->pLeft), .valNode = nodesCloneNode(pOper->pRight), .type = pOper->opType}; + .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), .type = pOper->opType}; taosArrayPush(sclOpList, &sclOp); } } @@ -4440,8 +4440,8 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); - nodesDestroyNode(sclOp->colNode); - nodesDestroyNode(sclOp->valNode); + nodesDestroyNode((SNode*)sclOp->colNode); + nodesDestroyNode((SNode*)sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 857cf3d269952a1f4a3010f01f67c2a8d77b9af4 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 14:29:33 +0800 Subject: [PATCH 13/28] fix: fix bugs --- source/libs/scalar/src/filter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index c1de62a407..2e0750359f 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3750,6 +3750,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t for (; j < numOfCols; ++j) { if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { foundCol = true; + break; } } if (foundCol) { From 22616d235ae120362973ab67cdbe3da0fabf2a0b Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 16:36:27 +0800 Subject: [PATCH 14/28] fix: fix minior bugs --- source/libs/scalar/src/filter.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 2e0750359f..e7f88908e5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3598,7 +3598,7 @@ int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { int32_t countRequired = (isUnion) ? 1 : 2; int32_t count = 0; for (int32_t i = 0; i < taosArrayGetSize(all); ++i) { - SFltSclPoint *pt = taosArrayGet(pts1, i); + SFltSclPoint *pt = taosArrayGet(all, i); if (pt->start) { ++count; if (count == countRequired) { @@ -4402,7 +4402,9 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = { - .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), .type = pOper->opType}; + .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; taosArrayPush(sclOpList, &sclOp); } } @@ -4433,7 +4435,7 @@ static int32_t fltSclCollectOperators(SNode *pNode, SArray *sclOpList) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { - SArray *sclOpList = taosArrayInit(16, POINTER_BYTES); + SArray *sclOpList = taosArrayInit(16, sizeof(SFltSclOperator)); fltSclCollectOperators(*pNode, sclOpList); SArray *colRangeList = taosArrayInit(16, sizeof(SFltSclColumnRange)); fltSclProcessCNF(sclOpList, colRangeList); From 2e4c14c18ce9a87f38eabe8c249fa26d04f70dbe Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 23 May 2023 21:30:41 +0800 Subject: [PATCH 15/28] fix: fix bugs --- source/libs/scalar/src/filter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index e7f88908e5..264e41b006 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3514,10 +3514,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { - if (val1->kind == FLT_SCL_DATUM_KIND_NULL || val1->kind == FLT_SCL_DATUM_KIND_MIN || - val1->kind == FLT_SCL_DATUM_KIND_MAX) { return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); - } } switch (val2->kind) { @@ -3690,7 +3687,12 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { - case TSDB_DATA_TYPE_BOOL: { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->i = val; break; @@ -3711,7 +3713,8 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int } // TODO:varchar/nchar/json default: { - qError("not supported"); + datum->kind = FLT_SCL_DATUM_KIND_NULL; + qError("not supported type when build datum from block sma value"); break; } } From 383c828c856df00a752b9843a615c468993f9ef3 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 11:18:44 +0800 Subject: [PATCH 16/28] fix: before review --- source/libs/scalar/src/filter.c | 90 +++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 264e41b006..ecf621a726 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3472,7 +3472,7 @@ int32_t fltSclCompareWithFloat64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3489,7 +3489,7 @@ int32_t fltSclCompareWithInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3506,7 +3506,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar, nchar default: - qError("not support comparsion. %d, %d", val1->kind, val2->kind); + qError("not supported comparsion. kind1 %d, kind2 %d", val1->kind, val2->kind); return (val1->kind - val2->kind); } } @@ -3514,7 +3514,7 @@ int32_t fltSclCompareWithUInt64(SFltSclDatum *val1, SFltSclDatum *val2) { int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { if (val2->kind == FLT_SCL_DATUM_KIND_NULL || val2->kind == FLT_SCL_DATUM_KIND_MIN || val2->kind == FLT_SCL_DATUM_KIND_MAX) { - return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); + return (val1->kind < val2->kind) ? -1 : ((val1->kind > val2->kind) ? 1 : 0); } switch (val2->kind) { @@ -3529,7 +3529,7 @@ int32_t fltSclCompareDatum(SFltSclDatum *val1, SFltSclDatum *val2) { } // TODO: varchar/nchar default: - fltError("not supported kind. just return 0"); + qError("not supported kind when compare datum. kind2 : %d", val2->kind); return 0; break; } @@ -3585,7 +3585,6 @@ int32_t fltSclMergeSort(SArray *pts1, SArray *pts2, SArray *result) { return 0; } -// pts1 and pts2 must be ordered and de-duplicated and each range can not be a range of another range int32_t fltSclMerge(SArray *pts1, SArray *pts2, bool isUnion, SArray *merged) { size_t len1 = taosArrayGetSize(pts1); size_t len2 = taosArrayGetSize(pts2); @@ -3616,7 +3615,6 @@ int32_t fltSclIntersect(SArray *pts1, SArray *pts2, SArray *merged) { return flt int32_t fltSclUnion(SArray *pts1, SArray *pts2, SArray *merged) { return fltSclMerge(pts1, pts2, true, merged); } -// TODO: column, constant typedef struct { SColumnNode *colNode; SValueNode *valNode; @@ -3675,9 +3673,9 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) datum->d = valNode->datum.d; break; } - // TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { - qError("not supported"); + qError("not supported type %d when build datum from value node", valNode->node.resType.type); break; } } @@ -3687,11 +3685,11 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode) int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int64_t val) { switch (type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->i = val; @@ -3711,10 +3709,10 @@ int32_t fltSclBuildDatumFromBlockSmaValue(SFltSclDatum *datum, uint8_t type, int datum->d = *(double *)&val; break; } - // TODO:varchar/nchar/json + // TODO:varchar/nchar/json default: { datum->kind = FLT_SCL_DATUM_KIND_NULL; - qError("not supported type when build datum from block sma value"); + qError("not supported type %d when build datum from block sma value", type); break; } } @@ -3732,6 +3730,13 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAg taosArrayPush(points, &endPt); return TSDB_CODE_SUCCESS; } + if (pAgg->numOfNull > 0) { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + } SFltSclDatum min; fltSclBuildDatumFromBlockSmaValue(&min, colRange->colNode->node.resType.type, pAgg->min); SFltSclPoint minPt = {.excl = false, .start = true, .val = min}; @@ -4357,8 +4362,25 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) { } break; } + case OP_TYPE_IS_NULL: { + SFltSclDatum nullDatum = {.kind = FLT_SCL_DATUM_KIND_NULL}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = nullDatum}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = nullDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } + case OP_TYPE_IS_NOT_NULL: { + SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_MIN, .type = oper->colNode->node.resType}; + SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum}; + SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_MAX, .type = oper->colNode->node.resType}; + SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum}; + taosArrayPush(points, &startPt); + taosArrayPush(points, &endPt); + break; + } default: { - qError("not supported op"); + qError("not supported operator type : %d when build range points", oper->type); break; } } @@ -4391,26 +4413,30 @@ static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { return TSDB_CODE_SUCCESS; } + SOperatorNode *pOper = (SOperatorNode *)pNode; - // TODO: left value node, right column node - // TODO: datatype - // TODO: operator if (pOper->pLeft == NULL || pOper->pRight == NULL) { return TSDB_CODE_SUCCESS; } - if (nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE && - (pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || - pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || - pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - SValueNode *valNode = (SValueNode *)pOper->pRight; - if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { - SFltSclOperator sclOp = { - .colNode = (SColumnNode*)nodesCloneNode(pOper->pLeft), - .valNode = (SValueNode*)nodesCloneNode(pOper->pRight), - .type = pOper->opType}; - taosArrayPush(sclOpList, &sclOp); - } + + if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || + pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || + pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { + return TSDB_CODE_SUCCESS; } + + if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return TSDB_CODE_SUCCESS; + } + + SValueNode *valNode = (SValueNode *)pOper->pRight; + if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { + SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), + .valNode = (SValueNode *)nodesCloneNode(pOper->pRight), + .type = pOper->opType}; + taosArrayPush(sclOpList, &sclOp); + } + return TSDB_CODE_SUCCESS; } @@ -4446,8 +4472,8 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) for (int32_t i = 0; i < taosArrayGetSize(sclOpList); ++i) { SFltSclOperator *sclOp = taosArrayGet(sclOpList, i); - nodesDestroyNode((SNode*)sclOp->colNode); - nodesDestroyNode((SNode*)sclOp->valNode); + nodesDestroyNode((SNode *)sclOp->colNode); + nodesDestroyNode((SNode *)sclOp->valNode); } taosArrayDestroy(sclOpList); return TSDB_CODE_SUCCESS; From 5b1e0be7560ebc658aec864e6db2b7fabcce15bb Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 13:41:21 +0800 Subject: [PATCH 17/28] enhance: add log --- source/libs/scalar/src/filter.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ecf621a726..3c01edf857 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3446,10 +3446,10 @@ typedef enum { typedef struct { SFltSclDatumKind kind; union { - int64_t i; // for int64, uint64 and double and bool (1 true, 0 false) - uint64_t u; - double d; - uint8_t *pData; // for varchar, nchar + int64_t i; // for int and bool (1 true, 0 false) and ts + uint64_t u; // for uint + double d; // for double + uint8_t *pData; // for varchar, nchar, len prefixed }; SDataType type; // TODO: original data type, may not be used? } SFltSclDatum; @@ -3765,9 +3765,15 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min " PRId64, pAgg->numOfNull, numOfRows, pAgg->max, + pAgg->min); + SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); bool isIntersect = taosArrayGetSize(merged) != 0; + qDebug("filter range execute, scalar mode, column range found. colId: %d colName: %s has overlap: %d", + colRange->colNode->colId, colRange->colNode->colName, isIntersect); + taosArrayDestroy(merged); taosArrayDestroy(points); if (!isIntersect) { From e5f9d78b6d0df0c0fa9970a29ce9f1d2617ae52a Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:39:38 +0800 Subject: [PATCH 18/28] fix: rocksdb so --- cmake/cmake.options | 5 -- contrib/CMakeLists.txt | 56 --------------------- contrib/test/CMakeLists.txt | 3 -- source/dnode/mgmt/mgmt_dnode/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 2 - source/libs/stream/CMakeLists.txt | 2 - 6 files changed, 1 insertion(+), 69 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index a7e62350df..064b1f7c6f 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -106,11 +106,6 @@ option( OFF ) -option( - BUILD_WITH_ROCKSDB - "If build with rocksdb" - ON -) option( BUILD_WITH_SQLITE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 536d4eae8e..ad3d8bc326 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,11 +77,6 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -if(${BUILD_WITH_ROCKSDB}) - cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) - add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -220,57 +215,6 @@ if(${BUILD_WITH_LEVELDB}) ) endif(${BUILD_WITH_LEVELDB}) -# rocksdb -# To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev -if(${BUILD_WITH_ROCKSDB}) - if(${TD_LINUX}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") - endif(${TD_LINUX}) - - if(${TD_DARWIN}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") - endif(${TD_DARWIN}) - - if (${TD_WINDOWS}) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") - endif(${TD_WINDOWS}) - - - if(${TD_DARWIN}) - option(HAVE_THREAD_LOCAL "" OFF) - option(WITH_IOSTATS_CONTEXT "" OFF) - option(WITH_PERF_CONTEXT "" OFF) - endif(${TD_DARWIN}) - - if(${TD_WINDOWS}) - option(WITH_JNI "" ON) - endif(${TD_WINDOWS}) - - if(${TD_WINDOWS}) - option(WITH_MD_LIBRARY "build with MD" OFF) - set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) - endif(${TD_WINDOWS}) - - - option(WITH_FALLOCATE "" OFF) - option(WITH_JEMALLOC "" OFF) - option(WITH_GFLAGS "" OFF) - option(PORTABLE "" ON) - option(WITH_LIBURING "" OFF) - option(FAIL_ON_WARNINGS OFF) - - option(WITH_TESTS "" OFF) - option(WITH_BENCHMARK_TOOLS "" OFF) - option(WITH_TOOLS "" OFF) - option(WITH_LIBURING "" OFF) - - option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb EXCLUDE_FROM_ALL) - target_include_directories( - rocksdb - PUBLIC $ - ) -endif(${BUILD_WITH_ROCKSDB}) # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index f35cf0d13d..8d095518e2 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -1,7 +1,4 @@ # rocksdb -if(${BUILD_WITH_ROCKSDB}) - add_subdirectory(rocksdb) -endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt index fdd0830a58..7d0b6d617d 100644 --- a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( PUBLIC "${GRANT_CFG_INCLUDE_DIR}" ) target_link_libraries( - mgmt_dnode node_util + mgmt_dnode node_util rocksdb ) \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index e8660cd6ad..79a7d55198 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -117,9 +117,7 @@ if(${BUILD_WITH_INVERTEDINDEX}) add_definitions(-DUSE_INVERTED_INDEX) endif(${BUILD_WITH_INVERTEDINDEX}) -if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 2edbc44aae..5952250b3e 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,7 +6,6 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb @@ -19,7 +18,6 @@ if(${BUILD_WITH_ROCKSDB}) ) add_definitions(-DUSE_ROCKSDB) -endif(${BUILD_WITH_ROCKSDB}) #target_link_libraries( From d7b1613d850eed2d65faf94c84f58a07320082b8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:42:50 +0800 Subject: [PATCH 19/28] Revert "fix: rocksdb so" This reverts commit e5f9d78b6d0df0c0fa9970a29ce9f1d2617ae52a. --- cmake/cmake.options | 5 ++ contrib/CMakeLists.txt | 56 +++++++++++++++++++++ contrib/test/CMakeLists.txt | 3 ++ source/dnode/mgmt/mgmt_dnode/CMakeLists.txt | 2 +- source/dnode/vnode/CMakeLists.txt | 2 + source/libs/stream/CMakeLists.txt | 2 + 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index 064b1f7c6f..a7e62350df 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -106,6 +106,11 @@ option( OFF ) +option( + BUILD_WITH_ROCKSDB + "If build with rocksdb" + ON +) option( BUILD_WITH_SQLITE diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ad3d8bc326..536d4eae8e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,6 +77,11 @@ if(${BUILD_WITH_LEVELDB}) cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_LEVELDB}) +# rocksdb +if(${BUILD_WITH_ROCKSDB}) + cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) @@ -215,6 +220,57 @@ if(${BUILD_WITH_LEVELDB}) ) endif(${BUILD_WITH_LEVELDB}) +# rocksdb +# To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev +if(${BUILD_WITH_ROCKSDB}) + if(${TD_LINUX}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result") + endif(${TD_LINUX}) + + if(${TD_DARWIN}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + endif(${TD_DARWIN}) + + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + endif(${TD_WINDOWS}) + + + if(${TD_DARWIN}) + option(HAVE_THREAD_LOCAL "" OFF) + option(WITH_IOSTATS_CONTEXT "" OFF) + option(WITH_PERF_CONTEXT "" OFF) + endif(${TD_DARWIN}) + + if(${TD_WINDOWS}) + option(WITH_JNI "" ON) + endif(${TD_WINDOWS}) + + if(${TD_WINDOWS}) + option(WITH_MD_LIBRARY "build with MD" OFF) + set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) + endif(${TD_WINDOWS}) + + + option(WITH_FALLOCATE "" OFF) + option(WITH_JEMALLOC "" OFF) + option(WITH_GFLAGS "" OFF) + option(PORTABLE "" ON) + option(WITH_LIBURING "" OFF) + option(FAIL_ON_WARNINGS OFF) + + option(WITH_TESTS "" OFF) + option(WITH_BENCHMARK_TOOLS "" OFF) + option(WITH_TOOLS "" OFF) + option(WITH_LIBURING "" OFF) + + option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) + target_include_directories( + rocksdb + PUBLIC $ + ) +endif(${BUILD_WITH_ROCKSDB}) # lucene # To support build on ubuntu: sudo apt-get install libboost-all-dev diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 8d095518e2..f35cf0d13d 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -1,4 +1,7 @@ # rocksdb +if(${BUILD_WITH_ROCKSDB}) + add_subdirectory(rocksdb) +endif(${BUILD_WITH_ROCKSDB}) if(${BUILD_WITH_LUCENE}) add_subdirectory(lucene) diff --git a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt index 7d0b6d617d..fdd0830a58 100644 --- a/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt +++ b/source/dnode/mgmt/mgmt_dnode/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( PUBLIC "${GRANT_CFG_INCLUDE_DIR}" ) target_link_libraries( - mgmt_dnode node_util rocksdb + mgmt_dnode node_util ) \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 79a7d55198..e8660cd6ad 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -117,7 +117,9 @@ if(${BUILD_WITH_INVERTEDINDEX}) add_definitions(-DUSE_INVERTED_INDEX) endif(${BUILD_WITH_INVERTEDINDEX}) +if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index 5952250b3e..2edbc44aae 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -6,6 +6,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +if(${BUILD_WITH_ROCKSDB}) target_link_libraries( stream PUBLIC rocksdb tdb @@ -18,6 +19,7 @@ target_include_directories( ) add_definitions(-DUSE_ROCKSDB) +endif(${BUILD_WITH_ROCKSDB}) #target_link_libraries( From 768372fc7bca72b43b47a4dc29ebc3b96dab4c6d Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 24 May 2023 16:45:06 +0800 Subject: [PATCH 20/28] fix: fix compilation warning --- source/libs/scalar/src/filter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 3c01edf857..27cf9da1d5 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,7 +3765,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min " PRId64, pAgg->numOfNull, numOfRows, pAgg->max, + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, pAgg->max, pAgg->min); SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); From de1848cb08726936958a51f6667ca33e49eb6995 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 14:51:53 +0800 Subject: [PATCH 21/28] enhance: use scalar mode to run ci test --- source/libs/function/src/udfd.c | 2 +- source/libs/scalar/src/filter.c | 2 +- source/libs/scalar/src/sclvector.c | 17 ++++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index b6ab7bd6f1..3b827a2f99 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -600,9 +600,9 @@ SUdf *udfdGetOrCreateUdf(const char *udfName) { return udf; } else { (*pUdfHash)->expired = true; - taosHashRemove(global.udfsHash, udfName, strlen(udfName)); fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64, (*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime); + taosHashRemove(global.udfsHash, udfName, strlen(udfName)); } } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 27cf9da1d5..464d388db4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -4556,7 +4556,7 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - info->scalarMode = stat.scalarMode; + info->scalarMode = true; fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4d803cb638..6983f6313b 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1784,6 +1784,9 @@ void vectorNotMatch(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOu void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1792,6 +1795,9 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t _ord) { for (int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; + if (v) { + ++pOut->numOfQualified; + } colDataSetInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1805,6 +1811,13 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, colDataSetInt8(pOut->columnData, i, &v); colDataClearNull_f(pOut->columnData->nullbitmap, i); } + { + bool v = false; + GET_TYPED_DATA(v, bool, pOut->columnData->info.type, colDataGetData(pOut->columnData, i)); + if (v) { + ++pOut->numOfQualified; + } + } } pOut->columnData->hasNull = false; } @@ -1844,7 +1857,9 @@ void vectorJsonContains(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam char *pLeftData = colDataGetVarData(pLeft->columnData, i); getJsonValue(pLeftData, jsonKey, &isExist); } - + if (isExist) { + ++pOut->numOfQualified; + } colDataSetVal(pOutputCol, i, (const char *)(&isExist), false); } taosMemoryFree(jsonKey); From 61b063529384019a954f05fdcb247dc8623537ed Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 16:27:48 +0800 Subject: [PATCH 22/28] fix: get time range using new scalar mode range --- source/libs/scalar/src/filter.c | 46 +++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 464d388db4..ae4712b9ba 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3765,8 +3765,8 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t SColumnDataAgg *pAgg = pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); - qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, pAgg->max, - pAgg->min); + qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, + pAgg->max, pAgg->min); SArray *merged = taosArrayInit(8, sizeof(SFltSclPoint)); fltSclIntersect(points, colRange->points, merged); @@ -3960,6 +3960,31 @@ _return: return code; } +static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d) { + *d = point->val; + if (point->val.kind == FLT_SCL_DATUM_KIND_NULL) { + return TSDB_CODE_SUCCESS; + } + if (point->val.kind == FLT_SCL_DATUM_KIND_MAX) { + getDataMax(d->type.type, &(d->i)); + } else if (point->val.kind == FLT_SCL_DATUM_KIND_MIN) { + getDataMin(d->type.type, &(d->i)); + } + + if ((point->val.kind == FLT_SCL_DATUM_KIND_INT64) || (point->val.kind = FLT_SCL_DATUM_KIND_UINT64)) { + if (point->excl) { + if (point->start) { + ++d->i; + } else { + --d->i; + } + } + } else { + qError("not supported kind %d when get datum from point", point->val.kind); + } + return TSDB_CODE_SUCCESS; +} + int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { SFilterInfo *info = NULL; int32_t code = 0; @@ -3969,6 +3994,23 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { FLT_ERR_RET(filterInitFromNode(pNode, &info, FLT_OPTION_NO_REWRITE | FLT_OPTION_TIMESTAMP)); if (info->scalarMode) { + SArray *colRanges = info->sclCtx.fltSclRange; + if (taosArrayGetSize(colRanges) == 1) { + SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); + SArray *points = colRange->points; + if (taosArrayGetSize(points) == 2) { + SFltSclPoint *startPt = taosArrayGet(points, 0); + SFltSclPoint *endPt = taosArrayGet(points, 1); + SFltSclDatum start; + SFltSclDatum end; + fltSclGetDatumValueFromPoint(startPt, &start); + fltSclGetDatumValueFromPoint(endPt, &end); + win->skey = start.i; + win->ekey = end.i; + *isStrict = true; + goto _return; + } + } *win = TSWINDOW_INITIALIZER; *isStrict = false; goto _return; From 469b2b573c1ac673cbe54307d3613eafa133319c Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 21:49:28 +0800 Subject: [PATCH 23/28] fix: fix error during emptyTsRange.sim --- source/libs/scalar/src/filter.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index ae4712b9ba..8a97f5fae2 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3546,7 +3546,7 @@ bool fltSclLessPoint(SFltSclPoint *pt1, SFltSclPoint *pt2) { if (pt1->start && pt2->start) { return !pt1->excl && pt2->excl; } else if (pt1->start) { - return pt1->excl && !pt2->excl; + return !pt1->excl && !pt2->excl; } else if (pt2->start) { return pt1->excl || pt2->excl; } @@ -3971,7 +3971,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d getDataMin(d->type.type, &(d->i)); } - if ((point->val.kind == FLT_SCL_DATUM_KIND_INT64) || (point->val.kind = FLT_SCL_DATUM_KIND_UINT64)) { + if (IS_NUMERIC_TYPE(d->type.type)) { if (point->excl) { if (point->start) { ++d->i; @@ -3980,7 +3980,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d } } } else { - qError("not supported kind %d when get datum from point", point->val.kind); + qError("not supported type %d when get datum from point", d->type.type); } return TSDB_CODE_SUCCESS; } @@ -4001,14 +4001,17 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) { if (taosArrayGetSize(points) == 2) { SFltSclPoint *startPt = taosArrayGet(points, 0); SFltSclPoint *endPt = taosArrayGet(points, 1); - SFltSclDatum start; - SFltSclDatum end; + SFltSclDatum start; + SFltSclDatum end; fltSclGetDatumValueFromPoint(startPt, &start); fltSclGetDatumValueFromPoint(endPt, &end); win->skey = start.i; win->ekey = end.i; *isStrict = true; goto _return; + } else if (taosArrayGetSize(points) == 0) { + *win = TSWINDOW_DESC_INITIALIZER; + goto _return; } } *win = TSWINDOW_INITIALIZER; From cbbd09b85c0fbe7038cfbfd3402888b217638acf Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 25 May 2023 22:37:49 +0800 Subject: [PATCH 24/28] fix: for logical and condition, each element shall be collectable operator --- source/libs/scalar/src/filter.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 8a97f5fae2..d45abb1b87 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3971,7 +3971,7 @@ static int32_t fltSclGetDatumValueFromPoint(SFltSclPoint *point, SFltSclDatum *d getDataMin(d->type.type, &(d->i)); } - if (IS_NUMERIC_TYPE(d->type.type)) { + if (IS_INTEGER_TYPE(d->type.type) || IS_TIMESTAMP_TYPE(d->type.type)) { if (point->excl) { if (point->start) { ++d->i; @@ -4460,26 +4460,35 @@ int32_t fltSclProcessCNF(SArray *sclOpListCNF, SArray *colRangeList) { return TSDB_CODE_SUCCESS; } -static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { +static bool fltSclIsCollectableNode(SNode *pNode) { if (nodeType(pNode) != QUERY_NODE_OPERATOR) { - return TSDB_CODE_SUCCESS; + return false; } SOperatorNode *pOper = (SOperatorNode *)pNode; if (pOper->pLeft == NULL || pOper->pRight == NULL) { - return TSDB_CODE_SUCCESS; + return false; } if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { - return TSDB_CODE_SUCCESS; + return false; } if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { + return false; + } + return true; +} + +static int32_t fltSclCollectOperatorFromNode(SNode *pNode, SArray *sclOpList) { + if (!fltSclIsCollectableNode(pNode)) { return TSDB_CODE_SUCCESS; } + SOperatorNode *pOper = (SOperatorNode *)pNode; + SValueNode *valNode = (SValueNode *)pOper->pRight; if (IS_NUMERIC_TYPE(valNode->node.resType.type) || valNode->node.resType.type == TSDB_DATA_TYPE_TIMESTAMP) { SFltSclOperator sclOp = {.colNode = (SColumnNode *)nodesCloneNode(pOper->pLeft), @@ -4501,6 +4510,11 @@ static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpLi return TSDB_CODE_SUCCESS; } SNode *pExpr = NULL; + FOREACH(pExpr, pLogicCond->pParameterList) { + if (!fltSclIsCollectableNode(pExpr)) { + return TSDB_CODE_SUCCESS; + } + } FOREACH(pExpr, pLogicCond->pParameterList) { fltSclCollectOperatorFromNode(pExpr, sclOpList); } return TSDB_CODE_SUCCESS; } From 81c796953e5233d1a2603eab13b71e7f25d12a0b Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 10:55:52 +0800 Subject: [PATCH 25/28] fix: address sanitizer error --- source/libs/executor/src/executorInt.c | 7 ++++++- source/libs/scalar/src/sclvector.c | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index d229f7d0ee..66ad3ca892 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -573,8 +573,13 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (colDataIsNull_var(pDst, j)) { colDataSetNull_var(pDst, numOfRows); } else { + // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - colDataSetVal(pDst, numOfRows, p1, false); + int32_t len = varDataTLen(p1); + char* p2 = taosMemoryMalloc(len); + varDataCopy(p2, p1); + colDataSetVal(pDst, numOfRows, p2, false); + taosMemoryFree(p2); } numOfRows += 1; j += 1; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 6983f6313b..e2496e15c3 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1788,6 +1788,7 @@ void vectorIsNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, ++pOut->numOfQualified; } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } @@ -1799,6 +1800,7 @@ void vectorNotNull(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut ++pOut->numOfQualified; } colDataSetInt8(pOut->columnData, i, &v); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } pOut->numOfRows = pLeft->numOfRows; } From 70bcbafbb8b3ee3adde1bd1813857fec9c9732da Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 13:36:56 +0800 Subject: [PATCH 26/28] fix: json value length is not same as binary/nchar length --- source/libs/executor/src/executorInt.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 66ad3ca892..ecd1874fa5 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -575,9 +575,14 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } else { // fix address sanitizer error. p1 may point to memory that will change during realloc of colDataSetVal, first copy it to p2 char* p1 = colDataGetVarData(pDst, j); - int32_t len = varDataTLen(p1); + int32_t len = 0; + if (pDst->info.type == TSDB_DATA_TYPE_JSON) { + len = getJsonValueLen(p1); + } else { + len = varDataTLen(p1); + } char* p2 = taosMemoryMalloc(len); - varDataCopy(p2, p1); + memcpy(p2, p1, len); colDataSetVal(pDst, numOfRows, p2, false); taosMemoryFree(p2); } From b83bf9163ec854d2fb857666fd50693df16572fa Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 14:46:32 +0800 Subject: [PATCH 27/28] fix: restore scalar mode computation and add filterScalarMode config variable and add test case --- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 5 + source/libs/scalar/src/filter.c | 10 +- tests/parallel_test/cases.task | 2 + tests/script/tsim/parser/condition_scl.sim | 136 +++++++++++++++++++ tests/script/tsim/query/emptyTsRange_scl.sim | 21 +++ 6 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 tests/script/tsim/parser/condition_scl.sim create mode 100644 tests/script/tsim/query/emptyTsRange_scl.sim diff --git a/include/common/tglobal.h b/include/common/tglobal.h index abee37d122..157e37f080 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -180,6 +180,7 @@ extern int32_t tsRpcRetryInterval; extern bool tsDisableStream; extern int64_t tsStreamBufferSize; extern int64_t tsCheckpointInterval; +extern bool tsFilterScalarMode; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9ff297c5b4..c648f8551a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -208,6 +208,7 @@ char tsUdfdLdLibPath[512] = ""; bool tsDisableStream = false; int64_t tsStreamBufferSize = 128 * 1024 * 1024; int64_t tsCheckpointInterval = 3 * 60 * 60 * 1000; +bool tsFilterScalarMode = false; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -522,6 +523,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, 0) != 0) return -1; + GRANT_CFG_ADD; return 0; } @@ -898,6 +901,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; tsCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i64; + tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; + GRANT_CFG_GET; return 0; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d45abb1b87..9a488edd67 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -14,6 +14,7 @@ */ #include #include "os.h" +#include "tglobal.h" #include "thash.h" // #include "queryLog.h" #include "filter.h" @@ -4510,7 +4511,7 @@ static int32_t fltSclCollectOperatorsFromLogicCond(SNode *pNode, SArray *sclOpLi return TSDB_CODE_SUCCESS; } SNode *pExpr = NULL; - FOREACH(pExpr, pLogicCond->pParameterList) { + FOREACH(pExpr, pLogicCond->pParameterList) { if (!fltSclIsCollectableNode(pExpr)) { return TSDB_CODE_SUCCESS; } @@ -4614,8 +4615,11 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) stat.info = info; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); - - info->scalarMode = true; + if (tsFilterScalarMode) { + info->scalarMode = true; + } else { + info->scalarMode = stat.scalarMode; + } fltDebug("scalar mode: %d", info->scalarMode); if (!info->scalarMode) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab4663e5b3..6037a14c8a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1274,6 +1274,7 @@ ,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim ,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim ,,y,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/condition_scl.sim ,,y,script,./test.sh -f tsim/parser/constCol.sim ,,y,script,./test.sh -f tsim/parser/create_db.sim ,,y,script,./test.sh -f tsim/parser/create_mt.sim @@ -1353,6 +1354,7 @@ ,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange_scl.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim ,,y,script,./test.sh -f tsim/query/tag_scan.sim diff --git a/tests/script/tsim/parser/condition_scl.sim b/tests/script/tsim/parser/condition_scl.sim new file mode 100644 index 0000000000..f377988006 --- /dev/null +++ b/tests/script/tsim/parser/condition_scl.sim @@ -0,0 +1,136 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists cdb +sql create database if not exists cdb +sql use cdb +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb1 using stb1 tags(1,'1',1.0) +sql create table tb2 using stb1 tags(2,'2',2.0) +sql create table tb3 using stb1 tags(3,'3',3.0) +sql create table tb4 using stb1 tags(4,'4',4.0) +sql create table tb5 using stb1 tags(5,'5',5.0) +sql create table tb6 using stb1 tags(6,'6',6.0) + +sql insert into tb1 values ('2021-05-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb1 values ('2021-05-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb1 values ('2021-05-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb1 values ('2021-05-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb1 values ('2021-05-05 18:19:04',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb1 values ('2021-05-05 18:19:05',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb1 values ('2021-05-05 18:19:06',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb1 values ('2021-05-05 18:19:07',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb2 values ('2021-05-05 18:19:08',21,21.0,21,21,21,21.0,true ,'21','21') +sql insert into tb2 values ('2021-05-05 18:19:09',22,22.0,22,22,22,22.0,true ,'22','22') +sql insert into tb2 values ('2021-05-05 18:19:10',23,23.0,23,23,23,23.0,false,'23','23') +sql insert into tb2 values ('2021-05-05 18:19:11',24,24.0,24,24,24,24.0,false,'24','24') +sql insert into tb3 values ('2021-05-05 18:19:12',31,31.0,31,31,31,31.0,true ,'31','31') +sql insert into tb3 values ('2021-05-05 18:19:13',32,32.0,32,32,32,32.0,true ,'32','32') +sql insert into tb3 values ('2021-05-05 18:19:14',33,33.0,33,33,33,33.0,false,'33','33') +sql insert into tb3 values ('2021-05-05 18:19:15',34,34.0,34,34,34,34.0,false,'34','34') +sql insert into tb4 values ('2021-05-05 18:19:16',41,41.0,41,41,41,41.0,true ,'41','41') +sql insert into tb4 values ('2021-05-05 18:19:17',42,42.0,42,42,42,42.0,true ,'42','42') +sql insert into tb4 values ('2021-05-05 18:19:18',43,43.0,43,43,43,43.0,false,'43','43') +sql insert into tb4 values ('2021-05-05 18:19:19',44,44.0,44,44,44,44.0,false,'44','44') +sql insert into tb5 values ('2021-05-05 18:19:20',51,51.0,51,51,51,51.0,true ,'51','51') +sql insert into tb5 values ('2021-05-05 18:19:21',52,52.0,52,52,52,52.0,true ,'52','52') +sql insert into tb5 values ('2021-05-05 18:19:22',53,53.0,53,53,53,53.0,false,'53','53') +sql insert into tb5 values ('2021-05-05 18:19:23',54,54.0,54,54,54,54.0,false,'54','54') +sql insert into tb6 values ('2021-05-05 18:19:24',61,61.0,61,61,61,61.0,true ,'61','61') +sql insert into tb6 values ('2021-05-05 18:19:25',62,62.0,62,62,62,62.0,true ,'62','62') +sql insert into tb6 values ('2021-05-05 18:19:26',63,63.0,63,63,63,63.0,false,'63','63') +sql insert into tb6 values ('2021-05-05 18:19:27',64,64.0,64,64,64,64.0,false,'64','64') +sql insert into tb6 values ('2021-05-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb2 (ts timestamp, u1 int unsigned, u2 bigint unsigned, u3 smallint unsigned, u4 tinyint unsigned, ts2 timestamp) TAGS(t1 int unsigned, t2 bigint unsigned, t3 timestamp, t4 int) +sql create table tb2_1 using stb2 tags(1,1,'2021-05-05 18:38:38',1) +sql create table tb2_2 using stb2 tags(2,2,'2021-05-05 18:58:58',2) + +sql insert into tb2_1 values ('2021-05-05 18:19:00',1,2,3,4,'2021-05-05 18:28:01') +sql insert into tb2_1 values ('2021-05-05 18:19:01',5,6,7,8,'2021-05-05 18:28:02') +sql insert into tb2_1 values ('2021-05-05 18:19:02',2,2,3,4,'2021-05-05 18:28:03') +sql insert into tb2_1 values ('2021-05-05 18:19:03',5,6,7,8,'2021-05-05 18:28:04') +sql insert into tb2_1 values ('2021-05-05 18:19:04',3,2,3,4,'2021-05-05 18:28:05') +sql insert into tb2_1 values ('2021-05-05 18:19:05',5,6,7,8,'2021-05-05 18:28:06') +sql insert into tb2_1 values ('2021-05-05 18:19:06',4,2,3,4,'2021-05-05 18:28:07') +sql insert into tb2_1 values ('2021-05-05 18:19:07',5,6,7,8,'2021-05-05 18:28:08') +sql insert into tb2_1 values ('2021-05-05 18:19:08',5,2,3,4,'2021-05-05 18:28:09') +sql insert into tb2_1 values ('2021-05-05 18:19:09',5,6,7,8,'2021-05-05 18:28:10') +sql insert into tb2_1 values ('2021-05-05 18:19:10',6,2,3,4,'2021-05-05 18:28:11') +sql insert into tb2_2 values ('2021-05-05 18:19:11',5,6,7,8,'2021-05-05 18:28:12') +sql insert into tb2_2 values ('2021-05-05 18:19:12',7,2,3,4,'2021-05-05 18:28:13') +sql insert into tb2_2 values ('2021-05-05 18:19:13',5,6,7,8,'2021-05-05 18:28:14') +sql insert into tb2_2 values ('2021-05-05 18:19:14',8,2,3,4,'2021-05-05 18:28:15') +sql insert into tb2_2 values ('2021-05-05 18:19:15',5,6,7,8,'2021-05-05 18:28:16') + +sql create table stb3 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb3_1 using stb3 tags(1,'1',1.0) +sql create table tb3_2 using stb3 tags(2,'2',2.0) + +sql insert into tb3_1 values ('2021-01-05 18:19:00',1,1.0,1,1,1,1.0,true ,'1','1') +sql insert into tb3_1 values ('2021-02-05 18:19:01',2,2.0,2,2,2,2.0,true ,'2','2') +sql insert into tb3_1 values ('2021-03-05 18:19:02',3,3.0,3,3,3,3.0,false,'3','3') +sql insert into tb3_1 values ('2021-04-05 18:19:03',4,4.0,4,4,4,4.0,false,'4','4') +sql insert into tb3_1 values ('2021-05-05 18:19:28',5,NULL,5,NULL,5,NULL,true,NULL,'5') +sql insert into tb3_1 values ('2021-06-05 18:19:28',NULL,6.0,NULL,6,NULL,6.0,NULL,'6',NULL) +sql insert into tb3_1 values ('2021-07-05 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) +sql insert into tb3_2 values ('2021-01-06 18:19:00',11,11.0,11,11,11,11.0,true ,'11','11') +sql insert into tb3_2 values ('2021-02-06 18:19:01',12,12.0,12,12,12,12.0,true ,'12','12') +sql insert into tb3_2 values ('2021-03-06 18:19:02',13,13.0,13,13,13,13.0,false,'13','13') +sql insert into tb3_2 values ('2021-04-06 18:19:03',14,14.0,14,14,14,14.0,false,'14','14') +sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true,NULL,'15') +sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL) +sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL) + +sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double) +sql create table tb4_0 using stb4 tags(0,'0',0.0) +sql create table tb4_1 using stb4 tags(1,'1',1.0) +sql create table tb4_2 using stb4 tags(2,'2',2.0) +sql create table tb4_3 using stb4 tags(3,'3',3.0) +sql create table tb4_4 using stb4 tags(4,'4',4.0) + +$i = 0 +$ts0 = 1625850000000 +$blockNum = 5 +$delta = 0 +$tbname0 = tb4_ +$a = 0 +$b = 200 +$c = 400 +while $i < $blockNum + $x = 0 + $rowNum = 1200 + while $x < $rowNum + $ts = $ts0 + $x + $a = $a + 1 + $b = $b + 1 + $c = $c + 1 + $d = $x / 10 + $tin = $rowNum + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + $tbname = 'tb4_ . $i + $tbname = $tbname . ' + sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary ) + $x = $x + 1 + endw + + $i = $i + 1 + $ts0 = $ts0 + 259200000 +endw + +run tsim/parser/condition_query.sim + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect + +run tsim/parser/condition_query.sim + diff --git a/tests/script/tsim/query/emptyTsRange_scl.sim b/tests/script/tsim/query/emptyTsRange_scl.sim new file mode 100644 index 0000000000..43734b047d --- /dev/null +++ b/tests/script/tsim/query/emptyTsRange_scl.sim @@ -0,0 +1,21 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c filterScalarMode -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database if not exists db1; +sql use db1; +sql create stable sta (ts timestamp, f1 double, f2 binary(200)) tags(t1 int); +sql create table tba1 using sta tags(1); +sql insert into tba1 values ('2022-04-26 15:15:01', 1.0, "a"); +sql insert into tba1 values ('2022-04-26 15:15:02', 2.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:04', 4.0, "b"); +sql insert into tba1 values ('2022-04-26 15:15:05', 5.0, "b"); +sql select last_row(*) from sta where ts >= 1678901803783 and ts <= 1678901803783 and _c0 <= 1678901803782 interval(10d,8d) fill(linear) order by _wstart desc; +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From b89f1652fcc16487de8e4fc5194c08316bc376dd Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 26 May 2023 14:52:14 +0800 Subject: [PATCH 28/28] fix: remove tbl count scan modification from the PR --- source/libs/planner/src/planOptimizer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4a803065ed..72e5081ab9 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2632,13 +2632,11 @@ static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pCondit return false; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { - return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions) && - LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); } if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { - return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions) && - LIST_LENGTH(pInfo->pAgg->pGroupKeys) == 0; + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); } return false;